From 2ee5f8f05949735fa2f4c463a5e13fcb3660c719 Mon Sep 17 00:00:00 2001 From: Daniel Lezcano Date: Tue, 8 Dec 2020 17:41:42 +0100 Subject: units: Add Watt units As there are the temperature units, let's add the Watt macros definition. Signed-off-by: Daniel Lezcano Reviewed-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- include/linux/units.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/units.h b/include/linux/units.h index aaf716364ec3..92c234e71cab 100644 --- a/include/linux/units.h +++ b/include/linux/units.h @@ -4,6 +4,10 @@ #include +#define MILLIWATT_PER_WATT 1000L +#define MICROWATT_PER_MILLIWATT 1000L +#define MICROWATT_PER_WATT 1000000L + #define ABSOLUTE_ZERO_MILLICELSIUS -273150 static inline long milli_kelvin_to_millicelsius(long t) -- cgit v1.2.3 From a20d0ef97abf486a917aff066c457bdb930425af Mon Sep 17 00:00:00 2001 From: Daniel Lezcano Date: Tue, 8 Dec 2020 17:41:44 +0100 Subject: powercap/drivers/dtpm: Add API for dynamic thermal power management On the embedded world, the complexity of the SoC leads to an increasing number of hotspots which need to be monitored and mitigated as a whole in order to prevent the temperature to go above the normative and legally stated 'skin temperature'. Another aspect is to sustain the performance for a given power budget, for example virtual reality where the user can feel dizziness if the GPU performance is capped while a big CPU is processing something else. Or reduce the battery charging because the dissipated power is too high compared with the power consumed by other devices. The userspace is the most adequate place to dynamically act on the different devices by limiting their power given an application profile: it has the knowledge of the platform. These userspace daemons are in charge of the Dynamic Thermal Power Management (DTPM). Nowadays, the dtpm daemons are abusing the thermal framework as they act on the cooling device state to force a specific and arbitrary state without taking care of the governor decisions. Given the closed loop of some governors that can confuse the logic or directly enter in a decision conflict. As the number of cooling device support is limited today to the CPU and the GPU, the dtpm daemons have little control on the power dissipation of the system. The out of tree solutions are hacking around here and there in the drivers, in the frameworks to have control on the devices. The common solution is to declare them as cooling devices. There is no unification of the power limitation unit, opaque states are used. This patch provides a way to create a hierarchy of constraints using the powercap framework. The devices which are registered as power limit-able devices are represented in this hierarchy as a tree. They are linked together with intermediate nodes which are just there to propagate the constraint to the children. The leaves of the tree are the real devices, the intermediate nodes are virtual, aggregating the children constraints and power characteristics. Each node have a weight on a 2^10 basis, in order to reflect the percentage of power distribution of the children's node. This percentage is used to dispatch the power limit to the children. The weight is computed against the max power of the siblings. This simple approach allows to do a fair distribution of the power limit. Signed-off-by: Daniel Lezcano Reviewed-by: Lukasz Luba Tested-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- drivers/powercap/Kconfig | 6 + drivers/powercap/Makefile | 1 + drivers/powercap/dtpm.c | 473 ++++++++++++++++++++++++++++++++++++++ include/asm-generic/vmlinux.lds.h | 11 + include/linux/dtpm.h | 75 ++++++ 5 files changed, 566 insertions(+) create mode 100644 drivers/powercap/dtpm.c create mode 100644 include/linux/dtpm.h (limited to 'include/linux') diff --git a/drivers/powercap/Kconfig b/drivers/powercap/Kconfig index bc228725346b..cc1953bd8bed 100644 --- a/drivers/powercap/Kconfig +++ b/drivers/powercap/Kconfig @@ -43,4 +43,10 @@ config IDLE_INJECT CPUs for power capping. Idle period can be injected synchronously on a set of specified CPUs or alternatively on a per CPU basis. + +config DTPM + bool "Power capping for Dynamic Thermal Power Management" + help + This enables support for the power capping for the dynamic + thermal power management userspace engine. endif diff --git a/drivers/powercap/Makefile b/drivers/powercap/Makefile index 7255c94ec61c..6482ac52054d 100644 --- a/drivers/powercap/Makefile +++ b/drivers/powercap/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_DTPM) += dtpm.o obj-$(CONFIG_POWERCAP) += powercap_sys.o obj-$(CONFIG_INTEL_RAPL_CORE) += intel_rapl_common.o obj-$(CONFIG_INTEL_RAPL) += intel_rapl_msr.o diff --git a/drivers/powercap/dtpm.c b/drivers/powercap/dtpm.c new file mode 100644 index 000000000000..5b6857e9b064 --- /dev/null +++ b/drivers/powercap/dtpm.c @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2020 Linaro Limited + * + * Author: Daniel Lezcano + * + * The powercap based Dynamic Thermal Power Management framework + * provides to the userspace a consistent API to set the power limit + * on some devices. + * + * DTPM defines the functions to create a tree of constraints. Each + * parent node is a virtual description of the aggregation of the + * children. It propagates the constraints set at its level to its + * children and collect the children power information. The leaves of + * the tree are the real devices which have the ability to get their + * current power consumption and set their power limit. + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +#define DTPM_POWER_LIMIT_FLAG BIT(0) + +static const char *constraint_name[] = { + "Instantaneous", +}; + +static DEFINE_MUTEX(dtpm_lock); +static struct powercap_control_type *pct; +static struct dtpm *root; + +static int get_time_window_us(struct powercap_zone *pcz, int cid, u64 *window) +{ + return -ENOSYS; +} + +static int set_time_window_us(struct powercap_zone *pcz, int cid, u64 window) +{ + return -ENOSYS; +} + +static int get_max_power_range_uw(struct powercap_zone *pcz, u64 *max_power_uw) +{ + struct dtpm *dtpm = to_dtpm(pcz); + + mutex_lock(&dtpm_lock); + *max_power_uw = dtpm->power_max - dtpm->power_min; + mutex_unlock(&dtpm_lock); + + return 0; +} + +static int __get_power_uw(struct dtpm *dtpm, u64 *power_uw) +{ + struct dtpm *child; + u64 power; + int ret = 0; + + if (dtpm->ops) { + *power_uw = dtpm->ops->get_power_uw(dtpm); + return 0; + } + + *power_uw = 0; + + list_for_each_entry(child, &dtpm->children, sibling) { + ret = __get_power_uw(child, &power); + if (ret) + break; + *power_uw += power; + } + + return ret; +} + +static int get_power_uw(struct powercap_zone *pcz, u64 *power_uw) +{ + struct dtpm *dtpm = to_dtpm(pcz); + int ret; + + mutex_lock(&dtpm_lock); + ret = __get_power_uw(dtpm, power_uw); + mutex_unlock(&dtpm_lock); + + return ret; +} + +static void __dtpm_rebalance_weight(struct dtpm *dtpm) +{ + struct dtpm *child; + + list_for_each_entry(child, &dtpm->children, sibling) { + + pr_debug("Setting weight '%d' for '%s'\n", + child->weight, child->zone.name); + + child->weight = DIV_ROUND_CLOSEST(child->power_max * 1024, + dtpm->power_max); + + __dtpm_rebalance_weight(child); + } +} + +static void __dtpm_sub_power(struct dtpm *dtpm) +{ + struct dtpm *parent = dtpm->parent; + + while (parent) { + parent->power_min -= dtpm->power_min; + parent->power_max -= dtpm->power_max; + parent->power_limit -= dtpm->power_limit; + parent = parent->parent; + } + + __dtpm_rebalance_weight(root); +} + +static void __dtpm_add_power(struct dtpm *dtpm) +{ + struct dtpm *parent = dtpm->parent; + + while (parent) { + parent->power_min += dtpm->power_min; + parent->power_max += dtpm->power_max; + parent->power_limit += dtpm->power_limit; + parent = parent->parent; + } + + __dtpm_rebalance_weight(root); +} + +/** + * dtpm_update_power - Update the power on the dtpm + * @dtpm: a pointer to a dtpm structure to update + * @power_min: a u64 representing the new power_min value + * @power_max: a u64 representing the new power_max value + * + * Function to update the power values of the dtpm node specified in + * parameter. These new values will be propagated to the tree. + * + * Return: zero on success, -EINVAL if the values are inconsistent + */ +int dtpm_update_power(struct dtpm *dtpm, u64 power_min, u64 power_max) +{ + mutex_lock(&dtpm_lock); + + if (power_min == dtpm->power_min && power_max == dtpm->power_max) + return 0; + + if (power_max < power_min) + return -EINVAL; + + __dtpm_sub_power(dtpm); + + dtpm->power_min = power_min; + dtpm->power_max = power_max; + if (!test_bit(DTPM_POWER_LIMIT_FLAG, &dtpm->flags)) + dtpm->power_limit = power_max; + + __dtpm_add_power(dtpm); + + mutex_unlock(&dtpm_lock); + + return 0; +} + +/** + * dtpm_release_zone - Cleanup when the node is released + * @pcz: a pointer to a powercap_zone structure + * + * Do some housecleaning and update the weight on the tree. The + * release will be denied if the node has children. This function must + * be called by the specific release callback of the different + * backends. + * + * Return: 0 on success, -EBUSY if there are children + */ +int dtpm_release_zone(struct powercap_zone *pcz) +{ + struct dtpm *dtpm = to_dtpm(pcz); + struct dtpm *parent = dtpm->parent; + + mutex_lock(&dtpm_lock); + + if (!list_empty(&dtpm->children)) + return -EBUSY; + + if (parent) + list_del(&dtpm->sibling); + + __dtpm_sub_power(dtpm); + + mutex_unlock(&dtpm_lock); + + if (dtpm->ops) + dtpm->ops->release(dtpm); + + kfree(dtpm); + + return 0; +} + +static int __get_power_limit_uw(struct dtpm *dtpm, int cid, u64 *power_limit) +{ + *power_limit = dtpm->power_limit; + return 0; +} + +static int get_power_limit_uw(struct powercap_zone *pcz, + int cid, u64 *power_limit) +{ + struct dtpm *dtpm = to_dtpm(pcz); + int ret; + + mutex_lock(&dtpm_lock); + ret = __get_power_limit_uw(dtpm, cid, power_limit); + mutex_unlock(&dtpm_lock); + + return ret; +} + +/* + * Set the power limit on the nodes, the power limit is distributed + * given the weight of the children. + * + * The dtpm node lock must be held when calling this function. + */ +static int __set_power_limit_uw(struct dtpm *dtpm, int cid, u64 power_limit) +{ + struct dtpm *child; + int ret = 0; + u64 power; + + /* + * A max power limitation means we remove the power limit, + * otherwise we set a constraint and flag the dtpm node. + */ + if (power_limit == dtpm->power_max) { + clear_bit(DTPM_POWER_LIMIT_FLAG, &dtpm->flags); + } else { + set_bit(DTPM_POWER_LIMIT_FLAG, &dtpm->flags); + } + + pr_debug("Setting power limit for '%s': %llu uW\n", + dtpm->zone.name, power_limit); + + /* + * Only leaves of the dtpm tree has ops to get/set the power + */ + if (dtpm->ops) { + dtpm->power_limit = dtpm->ops->set_power_uw(dtpm, power_limit); + } else { + dtpm->power_limit = 0; + + list_for_each_entry(child, &dtpm->children, sibling) { + + /* + * Integer division rounding will inevitably + * lead to a different min or max value when + * set several times. In order to restore the + * initial value, we force the child's min or + * max power every time if the constraint is + * at the boundaries. + */ + if (power_limit == dtpm->power_max) { + power = child->power_max; + } else if (power_limit == dtpm->power_min) { + power = child->power_min; + } else { + power = DIV_ROUND_CLOSEST( + power_limit * child->weight, 1024); + } + + pr_debug("Setting power limit for '%s': %llu uW\n", + child->zone.name, power); + + ret = __set_power_limit_uw(child, cid, power); + if (!ret) + ret = __get_power_limit_uw(child, cid, &power); + + if (ret) + break; + + dtpm->power_limit += power; + } + } + + return ret; +} + +static int set_power_limit_uw(struct powercap_zone *pcz, + int cid, u64 power_limit) +{ + struct dtpm *dtpm = to_dtpm(pcz); + int ret; + + mutex_lock(&dtpm_lock); + + /* + * Don't allow values outside of the power range previously + * set when initializing the power numbers. + */ + power_limit = clamp_val(power_limit, dtpm->power_min, dtpm->power_max); + + ret = __set_power_limit_uw(dtpm, cid, power_limit); + + pr_debug("%s: power limit: %llu uW, power max: %llu uW\n", + dtpm->zone.name, dtpm->power_limit, dtpm->power_max); + + mutex_unlock(&dtpm_lock); + + return ret; +} + +static const char *get_constraint_name(struct powercap_zone *pcz, int cid) +{ + return constraint_name[cid]; +} + +static int get_max_power_uw(struct powercap_zone *pcz, int id, u64 *max_power) +{ + struct dtpm *dtpm = to_dtpm(pcz); + + mutex_lock(&dtpm_lock); + *max_power = dtpm->power_max; + mutex_unlock(&dtpm_lock); + + return 0; +} + +static struct powercap_zone_constraint_ops constraint_ops = { + .set_power_limit_uw = set_power_limit_uw, + .get_power_limit_uw = get_power_limit_uw, + .set_time_window_us = set_time_window_us, + .get_time_window_us = get_time_window_us, + .get_max_power_uw = get_max_power_uw, + .get_name = get_constraint_name, +}; + +static struct powercap_zone_ops zone_ops = { + .get_max_power_range_uw = get_max_power_range_uw, + .get_power_uw = get_power_uw, + .release = dtpm_release_zone, +}; + +/** + * dtpm_alloc - Allocate and initialize a dtpm struct + * @name: a string specifying the name of the node + * + * Return: a struct dtpm pointer, NULL in case of error + */ +struct dtpm *dtpm_alloc(struct dtpm_ops *ops) +{ + struct dtpm *dtpm; + + dtpm = kzalloc(sizeof(*dtpm), GFP_KERNEL); + if (dtpm) { + INIT_LIST_HEAD(&dtpm->children); + INIT_LIST_HEAD(&dtpm->sibling); + dtpm->weight = 1024; + dtpm->ops = ops; + } + + return dtpm; +} + +/** + * dtpm_unregister - Unregister a dtpm node from the hierarchy tree + * @dtpm: a pointer to a dtpm structure corresponding to the node to be removed + * + * Call the underlying powercap unregister function. That will call + * the release callback of the powercap zone. + */ +void dtpm_unregister(struct dtpm *dtpm) +{ + powercap_unregister_zone(pct, &dtpm->zone); + + pr_info("Unregistered dtpm node '%s'\n", dtpm->zone.name); +} + +/** + * dtpm_register - Register a dtpm node in the hierarchy tree + * @name: a string specifying the name of the node + * @dtpm: a pointer to a dtpm structure corresponding to the new node + * @parent: a pointer to a dtpm structure corresponding to the parent node + * + * Create a dtpm node in the tree. If no parent is specified, the node + * is the root node of the hierarchy. If the root node already exists, + * then the registration will fail. The powercap controller must be + * initialized before calling this function. + * + * The dtpm structure must be initialized with the power numbers + * before calling this function. + * + * Return: zero on success, a negative value in case of error: + * -EAGAIN: the function is called before the framework is initialized. + * -EBUSY: the root node is already inserted + * -EINVAL: * there is no root node yet and @parent is specified + * * no all ops are defined + * * parent have ops which are reserved for leaves + * Other negative values are reported back from the powercap framework + */ +int dtpm_register(const char *name, struct dtpm *dtpm, struct dtpm *parent) +{ + struct powercap_zone *pcz; + + if (!pct) + return -EAGAIN; + + if (root && !parent) + return -EBUSY; + + if (!root && parent) + return -EINVAL; + + if (parent && parent->ops) + return -EINVAL; + + if (!dtpm) + return -EINVAL; + + if (dtpm->ops && !(dtpm->ops->set_power_uw && + dtpm->ops->get_power_uw && + dtpm->ops->release)) + return -EINVAL; + + pcz = powercap_register_zone(&dtpm->zone, pct, name, + parent ? &parent->zone : NULL, + &zone_ops, MAX_DTPM_CONSTRAINTS, + &constraint_ops); + if (IS_ERR(pcz)) + return PTR_ERR(pcz); + + mutex_lock(&dtpm_lock); + + if (parent) { + list_add_tail(&dtpm->sibling, &parent->children); + dtpm->parent = parent; + } else { + root = dtpm; + } + + __dtpm_add_power(dtpm); + + pr_info("Registered dtpm node '%s' / %llu-%llu uW, \n", + dtpm->zone.name, dtpm->power_min, dtpm->power_max); + + mutex_unlock(&dtpm_lock); + + return 0; +} + +static int __init dtpm_init(void) +{ + struct dtpm_descr **dtpm_descr; + + pct = powercap_register_control_type(NULL, "dtpm", NULL); + if (!pct) { + pr_err("Failed to register control type\n"); + return -EINVAL; + } + + for_each_dtpm_table(dtpm_descr) + (*dtpm_descr)->init(*dtpm_descr); + + return 0; +} +late_initcall(dtpm_init); diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index b2b3d81b1535..b3e4e0740089 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -316,6 +316,16 @@ #define THERMAL_TABLE(name) #endif +#ifdef CONFIG_DTPM +#define DTPM_TABLE() \ + . = ALIGN(8); \ + __dtpm_table = .; \ + KEEP(*(__dtpm_table)) \ + __dtpm_table_end = .; +#else +#define DTPM_TABLE() +#endif + #define KERNEL_DTB() \ STRUCT_ALIGN(); \ __dtb_start = .; \ @@ -733,6 +743,7 @@ ACPI_PROBE_TABLE(irqchip) \ ACPI_PROBE_TABLE(timer) \ THERMAL_TABLE(governor) \ + DTPM_TABLE() \ EARLYCON_TABLE() \ LSM_TABLE() \ EARLY_LSM_TABLE() \ diff --git a/include/linux/dtpm.h b/include/linux/dtpm.h new file mode 100644 index 000000000000..7a1d0b50e334 --- /dev/null +++ b/include/linux/dtpm.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Linaro Ltd + * + * Author: Daniel Lezcano + */ +#ifndef ___DTPM_H__ +#define ___DTPM_H__ + +#include + +#define MAX_DTPM_DESCR 8 +#define MAX_DTPM_CONSTRAINTS 1 + +struct dtpm { + struct powercap_zone zone; + struct dtpm *parent; + struct list_head sibling; + struct list_head children; + struct dtpm_ops *ops; + unsigned long flags; + u64 power_limit; + u64 power_max; + u64 power_min; + int weight; + void *private; +}; + +struct dtpm_ops { + u64 (*set_power_uw)(struct dtpm *, u64); + u64 (*get_power_uw)(struct dtpm *); + void (*release)(struct dtpm *); +}; + +struct dtpm_descr; + +typedef int (*dtpm_init_t)(struct dtpm_descr *); + +struct dtpm_descr { + struct dtpm *parent; + const char *name; + dtpm_init_t init; +}; + +/* Init section thermal table */ +extern struct dtpm_descr *__dtpm_table[]; +extern struct dtpm_descr *__dtpm_table_end[]; + +#define DTPM_TABLE_ENTRY(name) \ + static typeof(name) *__dtpm_table_entry_##name \ + __used __section("__dtpm_table") = &name + +#define DTPM_DECLARE(name) DTPM_TABLE_ENTRY(name) + +#define for_each_dtpm_table(__dtpm) \ + for (__dtpm = __dtpm_table; \ + __dtpm < __dtpm_table_end; \ + __dtpm++) + +static inline struct dtpm *to_dtpm(struct powercap_zone *zone) +{ + return container_of(zone, struct dtpm, zone); +} + +int dtpm_update_power(struct dtpm *dtpm, u64 power_min, u64 power_max); + +int dtpm_release_zone(struct powercap_zone *pcz); + +struct dtpm *dtpm_alloc(struct dtpm_ops *ops); + +void dtpm_unregister(struct dtpm *dtpm); + +int dtpm_register(const char *name, struct dtpm *dtpm, struct dtpm *parent); + +#endif -- cgit v1.2.3 From 0e8f68d7f04856a9e2ad4817b477fa35124888bd Mon Sep 17 00:00:00 2001 From: Daniel Lezcano Date: Tue, 8 Dec 2020 17:41:45 +0100 Subject: powercap/drivers/dtpm: Add CPU energy model based support With the powercap dtpm controller, we are able to plug devices with power limitation features in the tree. The following patch introduces the CPU power limitation based on the energy model and the performance states. The power limitation is done at the performance domain level. If some CPUs are unplugged, the corresponding power will be subtracted from the performance domain total power. It is up to the platform to initialize the dtpm tree and add the CPU. Here is an example to create a simple tree with one root node called "pkg" and the CPU's performance domains. static int dtpm_register_pkg(struct dtpm_descr *descr) { struct dtpm *pkg; int ret; pkg = dtpm_alloc(NULL); if (!pkg) return -ENOMEM; ret = dtpm_register(descr->name, pkg, descr->parent); if (ret) return ret; return dtpm_register_cpu(pkg); } static struct dtpm_descr descr = { .name = "pkg", .init = dtpm_register_pkg, }; DTPM_DECLARE(descr); Signed-off-by: Daniel Lezcano Reviewed-by: Lukasz Luba Tested-by: Lukasz Luba Signed-off-by: Rafael J. Wysocki --- drivers/powercap/Kconfig | 7 ++ drivers/powercap/Makefile | 1 + drivers/powercap/dtpm_cpu.c | 257 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/cpuhotplug.h | 1 + include/linux/dtpm.h | 2 + 5 files changed, 268 insertions(+) create mode 100644 drivers/powercap/dtpm_cpu.c (limited to 'include/linux') diff --git a/drivers/powercap/Kconfig b/drivers/powercap/Kconfig index cc1953bd8bed..20b4325c6161 100644 --- a/drivers/powercap/Kconfig +++ b/drivers/powercap/Kconfig @@ -49,4 +49,11 @@ config DTPM help This enables support for the power capping for the dynamic thermal power management userspace engine. + +config DTPM_CPU + bool "Add CPU power capping based on the energy model" + depends on DTPM && ENERGY_MODEL + help + This enables support for CPU power limitation based on + energy model. endif diff --git a/drivers/powercap/Makefile b/drivers/powercap/Makefile index 6482ac52054d..fabcf388a8d3 100644 --- a/drivers/powercap/Makefile +++ b/drivers/powercap/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_DTPM) += dtpm.o +obj-$(CONFIG_DTPM_CPU) += dtpm_cpu.o obj-$(CONFIG_POWERCAP) += powercap_sys.o obj-$(CONFIG_INTEL_RAPL_CORE) += intel_rapl_common.o obj-$(CONFIG_INTEL_RAPL) += intel_rapl_msr.o diff --git a/drivers/powercap/dtpm_cpu.c b/drivers/powercap/dtpm_cpu.c new file mode 100644 index 000000000000..6933c783c6b4 --- /dev/null +++ b/drivers/powercap/dtpm_cpu.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2020 Linaro Limited + * + * Author: Daniel Lezcano + * + * The DTPM CPU is based on the energy model. It hooks the CPU in the + * DTPM tree which in turns update the power number by propagating the + * power number from the CPU energy model information to the parents. + * + * The association between the power and the performance state, allows + * to set the power of the CPU at the OPP granularity. + * + * The CPU hotplug is supported and the power numbers will be updated + * if a CPU is hot plugged / unplugged. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +static struct dtpm *__parent; + +static DEFINE_PER_CPU(struct dtpm *, dtpm_per_cpu); + +struct dtpm_cpu { + struct freq_qos_request qos_req; + int cpu; +}; + +/* + * When a new CPU is inserted at hotplug or boot time, add the power + * contribution and update the dtpm tree. + */ +static int power_add(struct dtpm *dtpm, struct em_perf_domain *em) +{ + u64 power_min, power_max; + + power_min = em->table[0].power; + power_min *= MICROWATT_PER_MILLIWATT; + power_min += dtpm->power_min; + + power_max = em->table[em->nr_perf_states - 1].power; + power_max *= MICROWATT_PER_MILLIWATT; + power_max += dtpm->power_max; + + return dtpm_update_power(dtpm, power_min, power_max); +} + +/* + * When a CPU is unplugged, remove its power contribution from the + * dtpm tree. + */ +static int power_sub(struct dtpm *dtpm, struct em_perf_domain *em) +{ + u64 power_min, power_max; + + power_min = em->table[0].power; + power_min *= MICROWATT_PER_MILLIWATT; + power_min = dtpm->power_min - power_min; + + power_max = em->table[em->nr_perf_states - 1].power; + power_max *= MICROWATT_PER_MILLIWATT; + power_max = dtpm->power_max - power_max; + + return dtpm_update_power(dtpm, power_min, power_max); +} + +static u64 set_pd_power_limit(struct dtpm *dtpm, u64 power_limit) +{ + struct dtpm_cpu *dtpm_cpu = dtpm->private; + struct em_perf_domain *pd; + struct cpumask cpus; + unsigned long freq; + u64 power; + int i, nr_cpus; + + pd = em_cpu_get(dtpm_cpu->cpu); + + cpumask_and(&cpus, cpu_online_mask, to_cpumask(pd->cpus)); + + nr_cpus = cpumask_weight(&cpus); + + for (i = 0; i < pd->nr_perf_states; i++) { + + power = pd->table[i].power * MICROWATT_PER_MILLIWATT * nr_cpus; + + if (power > power_limit) + break; + } + + freq = pd->table[i - 1].frequency; + + freq_qos_update_request(&dtpm_cpu->qos_req, freq); + + power_limit = pd->table[i - 1].power * + MICROWATT_PER_MILLIWATT * nr_cpus; + + return power_limit; +} + +static u64 get_pd_power_uw(struct dtpm *dtpm) +{ + struct dtpm_cpu *dtpm_cpu = dtpm->private; + struct em_perf_domain *pd; + struct cpumask cpus; + unsigned long freq; + int i, nr_cpus; + + pd = em_cpu_get(dtpm_cpu->cpu); + freq = cpufreq_quick_get(dtpm_cpu->cpu); + cpumask_and(&cpus, cpu_online_mask, to_cpumask(pd->cpus)); + nr_cpus = cpumask_weight(&cpus); + + for (i = 0; i < pd->nr_perf_states; i++) { + + if (pd->table[i].frequency < freq) + continue; + + return pd->table[i].power * + MICROWATT_PER_MILLIWATT * nr_cpus; + } + + return 0; +} + +static void pd_release(struct dtpm *dtpm) +{ + struct dtpm_cpu *dtpm_cpu = dtpm->private; + + if (freq_qos_request_active(&dtpm_cpu->qos_req)) + freq_qos_remove_request(&dtpm_cpu->qos_req); + + kfree(dtpm_cpu); +} + +static struct dtpm_ops dtpm_ops = { + .set_power_uw = set_pd_power_limit, + .get_power_uw = get_pd_power_uw, + .release = pd_release, +}; + +static int cpuhp_dtpm_cpu_offline(unsigned int cpu) +{ + struct cpufreq_policy *policy; + struct em_perf_domain *pd; + struct dtpm *dtpm; + + policy = cpufreq_cpu_get(cpu); + + if (!policy) + return 0; + + pd = em_cpu_get(cpu); + if (!pd) + return -EINVAL; + + dtpm = per_cpu(dtpm_per_cpu, cpu); + + power_sub(dtpm, pd); + + if (cpumask_weight(policy->cpus) != 1) + return 0; + + for_each_cpu(cpu, policy->related_cpus) + per_cpu(dtpm_per_cpu, cpu) = NULL; + + dtpm_unregister(dtpm); + + return 0; +} + +static int cpuhp_dtpm_cpu_online(unsigned int cpu) +{ + struct dtpm *dtpm; + struct dtpm_cpu *dtpm_cpu; + struct cpufreq_policy *policy; + struct em_perf_domain *pd; + char name[CPUFREQ_NAME_LEN]; + int ret = -ENOMEM; + + policy = cpufreq_cpu_get(cpu); + + if (!policy) + return 0; + + pd = em_cpu_get(cpu); + if (!pd) + return -EINVAL; + + dtpm = per_cpu(dtpm_per_cpu, cpu); + if (dtpm) + return power_add(dtpm, pd); + + dtpm = dtpm_alloc(&dtpm_ops); + if (!dtpm) + return -EINVAL; + + dtpm_cpu = kzalloc(sizeof(dtpm_cpu), GFP_KERNEL); + if (!dtpm_cpu) + goto out_kfree_dtpm; + + dtpm->private = dtpm_cpu; + dtpm_cpu->cpu = cpu; + + for_each_cpu(cpu, policy->related_cpus) + per_cpu(dtpm_per_cpu, cpu) = dtpm; + + sprintf(name, "cpu%d", dtpm_cpu->cpu); + + ret = dtpm_register(name, dtpm, __parent); + if (ret) + goto out_kfree_dtpm_cpu; + + ret = power_add(dtpm, pd); + if (ret) + goto out_dtpm_unregister; + + ret = freq_qos_add_request(&policy->constraints, + &dtpm_cpu->qos_req, FREQ_QOS_MAX, + pd->table[pd->nr_perf_states - 1].frequency); + if (ret) + goto out_power_sub; + + return 0; + +out_power_sub: + power_sub(dtpm, pd); + +out_dtpm_unregister: + dtpm_unregister(dtpm); + dtpm_cpu = NULL; + dtpm = NULL; + +out_kfree_dtpm_cpu: + for_each_cpu(cpu, policy->related_cpus) + per_cpu(dtpm_per_cpu, cpu) = NULL; + kfree(dtpm_cpu); + +out_kfree_dtpm: + kfree(dtpm); + return ret; +} + +int dtpm_register_cpu(struct dtpm *parent) +{ + __parent = parent; + + return cpuhp_setup_state(CPUHP_AP_DTPM_CPU_ONLINE, + "dtpm_cpu:online", + cpuhp_dtpm_cpu_online, + cpuhp_dtpm_cpu_offline); +} diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 0042ef362511..ee09a39627d6 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -193,6 +193,7 @@ enum cpuhp_state { CPUHP_AP_ONLINE_DYN_END = CPUHP_AP_ONLINE_DYN + 30, CPUHP_AP_X86_HPET_ONLINE, CPUHP_AP_X86_KVM_CLK_ONLINE, + CPUHP_AP_DTPM_CPU_ONLINE, CPUHP_AP_ACTIVE, CPUHP_ONLINE, }; diff --git a/include/linux/dtpm.h b/include/linux/dtpm.h index 7a1d0b50e334..e80a332e3d8a 100644 --- a/include/linux/dtpm.h +++ b/include/linux/dtpm.h @@ -72,4 +72,6 @@ void dtpm_unregister(struct dtpm *dtpm); int dtpm_register(const char *name, struct dtpm *dtpm, struct dtpm *parent); +int dtpm_register_cpu(struct dtpm *parent); + #endif -- cgit v1.2.3 From 4ea3cd65e0d47c4d3fc0c86d2d93d97457dc86fb Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Fri, 18 Dec 2020 11:42:44 +0100 Subject: tty: rename tty_kopen() and add new function tty_kopen_shared() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a new function tty_kopen_shared() that yields a struct tty_struct. The semantic difference to tty_kopen() is that the tty is expected to be used already. So rename tty_kopen() to tty_kopen_exclusive() for clearness, adapt the single user and put the common code in a new static helper function. tty_kopen_shared is to be used to implement an LED trigger for tty devices in one of the next patches. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20201218104246.591315-2-u.kleine-koenig@pengutronix.de Signed-off-by: Greg Kroah-Hartman --- drivers/accessibility/speakup/spk_ttyio.c | 2 +- drivers/tty/tty_io.c | 56 +++++++++++++++++++++---------- include/linux/tty.h | 5 +-- 3 files changed, 42 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/accessibility/speakup/spk_ttyio.c b/drivers/accessibility/speakup/spk_ttyio.c index 6284aff434a1..835d17455fcd 100644 --- a/drivers/accessibility/speakup/spk_ttyio.c +++ b/drivers/accessibility/speakup/spk_ttyio.c @@ -152,7 +152,7 @@ static int spk_ttyio_initialise_ldisc(struct spk_synth *synth) if (ret) return ret; - tty = tty_kopen(dev); + tty = tty_kopen_exclusive(dev); if (IS_ERR(tty)) return PTR_ERR(tty); diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 8034489337d7..62ccd021102d 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -1875,22 +1875,7 @@ static struct tty_driver *tty_lookup_driver(dev_t device, struct file *filp, return driver; } -/** - * tty_kopen - open a tty device for kernel - * @device: dev_t of device to open - * - * Opens tty exclusively for kernel. Performs the driver lookup, - * makes sure it's not already opened and performs the first-time - * tty initialization. - * - * Returns the locked initialized &tty_struct - * - * Claims the global tty_mutex to serialize: - * - concurrent first-time tty initialization - * - concurrent tty driver removal w/ lookup - * - concurrent tty removal from driver table - */ -struct tty_struct *tty_kopen(dev_t device) +static struct tty_struct *tty_kopen(dev_t device, int shared) { struct tty_struct *tty; struct tty_driver *driver; @@ -1905,7 +1890,7 @@ struct tty_struct *tty_kopen(dev_t device) /* check whether we're reopening an existing tty */ tty = tty_driver_lookup_tty(driver, NULL, index); - if (IS_ERR(tty)) + if (IS_ERR(tty) || shared) goto out; if (tty) { @@ -1923,7 +1908,42 @@ out: tty_driver_kref_put(driver); return tty; } -EXPORT_SYMBOL_GPL(tty_kopen); + +/** + * tty_kopen_exclusive - open a tty device for kernel + * @device: dev_t of device to open + * + * Opens tty exclusively for kernel. Performs the driver lookup, + * makes sure it's not already opened and performs the first-time + * tty initialization. + * + * Returns the locked initialized &tty_struct + * + * Claims the global tty_mutex to serialize: + * - concurrent first-time tty initialization + * - concurrent tty driver removal w/ lookup + * - concurrent tty removal from driver table + */ +struct tty_struct *tty_kopen_exclusive(dev_t device) +{ + return tty_kopen(device, 0); +} +EXPORT_SYMBOL_GPL(tty_kopen_exclusive); + +/** + * tty_kopen_shared - open a tty device for shared in-kernel use + * @device: dev_t of device to open + * + * Opens an already existing tty for in-kernel use. Compared to + * tty_kopen_exclusive() above it doesn't ensure to be the only user. + * + * Locking is identical to tty_kopen() above. + */ +struct tty_struct *tty_kopen_shared(dev_t device) +{ + return tty_kopen(device, 1); +} +EXPORT_SYMBOL_GPL(tty_kopen_shared); /** * tty_open_by_driver - open a tty device diff --git a/include/linux/tty.h b/include/linux/tty.h index c873f475f0a7..df707a5abfac 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -416,7 +416,8 @@ extern struct tty_struct *get_current_tty(void); /* tty_io.c */ extern int __init tty_init(void); extern const char *tty_name(const struct tty_struct *tty); -extern struct tty_struct *tty_kopen(dev_t device); +extern struct tty_struct *tty_kopen_exclusive(dev_t device); +extern struct tty_struct *tty_kopen_shared(dev_t device); extern void tty_kclose(struct tty_struct *tty); extern int tty_dev_name_to_number(const char *name, dev_t *number); extern int tty_ldisc_lock(struct tty_struct *tty, unsigned long timeout); @@ -441,7 +442,7 @@ static inline int __init tty_init(void) { return 0; } static inline const char *tty_name(const struct tty_struct *tty) { return "(none)"; } -static inline struct tty_struct *tty_kopen(dev_t device) +static inline struct tty_struct *tty_kopen_exclusive(dev_t device) { return ERR_PTR(-ENODEV); } static inline void tty_kclose(struct tty_struct *tty) { } -- cgit v1.2.3 From d20c219c7317843cec7f03eba15bfeae54f29654 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Fri, 18 Dec 2020 11:42:45 +0100 Subject: tty: new helper function tty_get_icount() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For a given struct tty_struct this yields the corresponding statistics about sent and received characters (and some more) which is needed to implement an LED trigger for tty devices. The new function is then used to simplify tty_tiocgicount(). Reviewed-by: Pavel Machek Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20201218104246.591315-3-u.kleine-koenig@pengutronix.de Signed-off-by: Greg Kroah-Hartman --- drivers/tty/tty_io.c | 29 +++++++++++++++++++++++++---- include/linux/tty.h | 2 ++ 2 files changed, 27 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 62ccd021102d..95ba028ef668 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -2488,15 +2488,36 @@ static int tty_tiocmset(struct tty_struct *tty, unsigned int cmd, return tty->ops->tiocmset(tty, set, clear); } +/** + * tty_get_icount - get tty statistics + * @tty: tty device + * @icount: output parameter + * + * Gets a copy of the tty's icount statistics. + * + * Locking: none (up to the driver) + */ +int tty_get_icount(struct tty_struct *tty, + struct serial_icounter_struct *icount) +{ + memset(icount, 0, sizeof(*icount)); + + if (tty->ops->get_icount) + return tty->ops->get_icount(tty, icount); + else + return -EINVAL; +} +EXPORT_SYMBOL_GPL(tty_get_icount); + static int tty_tiocgicount(struct tty_struct *tty, void __user *arg) { - int retval = -EINVAL; struct serial_icounter_struct icount; - memset(&icount, 0, sizeof(icount)); - if (tty->ops->get_icount) - retval = tty->ops->get_icount(tty, &icount); + int retval; + + retval = tty_get_icount(tty, &icount); if (retval != 0) return retval; + if (copy_to_user(arg, &icount, sizeof(icount))) return -EFAULT; return 0; diff --git a/include/linux/tty.h b/include/linux/tty.h index df707a5abfac..12be8b16cdef 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -500,6 +500,8 @@ extern void tty_unthrottle(struct tty_struct *tty); extern int tty_throttle_safe(struct tty_struct *tty); extern int tty_unthrottle_safe(struct tty_struct *tty); extern int tty_do_resize(struct tty_struct *tty, struct winsize *ws); +extern int tty_get_icount(struct tty_struct *tty, + struct serial_icounter_struct *icount); extern int is_current_pgrp_orphaned(void); extern void tty_hangup(struct tty_struct *tty); extern void tty_vhangup(struct tty_struct *tty); -- cgit v1.2.3 From 916c0c05521a52f13283ad3600793fc79516ff31 Mon Sep 17 00:00:00 2001 From: Sai Prakash Ranjan Date: Mon, 30 Nov 2020 15:09:23 +0530 Subject: soc: qcom: llcc-qcom: Extract major hardware version The major hardware version of the LLCC IP is encoded in its LLCC_COMMON_HW_INFO register. Extract the version and cache it in the driver data so that it can be used to implement version specific functionality like enabling Write sub cache for given SCID. Signed-off-by: Sai Prakash Ranjan [mani: splitted the version extract as a single patch and few cleanups] Signed-off-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/20201130093924.45057-4-manivannan.sadhasivam@linaro.org Signed-off-by: Bjorn Andersson --- drivers/soc/qcom/llcc-qcom.c | 12 ++++++++++++ include/linux/soc/qcom/llcc-qcom.h | 2 ++ 2 files changed, 14 insertions(+) (limited to 'include/linux') diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c index 16b421608e9c..a559617ea7c0 100644 --- a/drivers/soc/qcom/llcc-qcom.c +++ b/drivers/soc/qcom/llcc-qcom.c @@ -4,6 +4,7 @@ * */ +#include #include #include #include @@ -35,6 +36,9 @@ #define CACHE_LINE_SIZE_SHIFT 6 +#define LLCC_COMMON_HW_INFO 0x00030000 +#define LLCC_MAJOR_VERSION_MASK GENMASK(31, 24) + #define LLCC_COMMON_STATUS0 0x0003000c #define LLCC_LB_CNT_MASK GENMASK(31, 28) #define LLCC_LB_CNT_SHIFT 28 @@ -476,6 +480,7 @@ static int qcom_llcc_probe(struct platform_device *pdev) const struct qcom_llcc_config *cfg; const struct llcc_slice_config *llcc_cfg; u32 sz; + u32 version; drv_data = devm_kzalloc(dev, sizeof(*drv_data), GFP_KERNEL); if (!drv_data) { @@ -496,6 +501,13 @@ static int qcom_llcc_probe(struct platform_device *pdev) goto err; } + /* Extract major version of the IP */ + ret = regmap_read(drv_data->bcast_regmap, LLCC_COMMON_HW_INFO, &version); + if (ret) + goto err; + + drv_data->major_version = FIELD_GET(LLCC_MAJOR_VERSION_MASK, version); + ret = regmap_read(drv_data->regmap, LLCC_COMMON_STATUS0, &num_banks); if (ret) diff --git a/include/linux/soc/qcom/llcc-qcom.h b/include/linux/soc/qcom/llcc-qcom.h index 3db6797ba6ff..d17a3de80510 100644 --- a/include/linux/soc/qcom/llcc-qcom.h +++ b/include/linux/soc/qcom/llcc-qcom.h @@ -79,6 +79,7 @@ struct llcc_edac_reg_data { * @bitmap: Bit map to track the active slice ids * @offsets: Pointer to the bank offsets array * @ecc_irq: interrupt for llcc cache error detection and reporting + * @major_version: Indicates the LLCC major version */ struct llcc_drv_data { struct regmap *regmap; @@ -91,6 +92,7 @@ struct llcc_drv_data { unsigned long *bitmap; u32 *offsets; int ecc_irq; + u32 major_version; }; #if IS_ENABLED(CONFIG_QCOM_LLCC) -- cgit v1.2.3 From c4df37fe186de4df8895a7a4793f5221eda6e5ae Mon Sep 17 00:00:00 2001 From: Manivannan Sadhasivam Date: Mon, 30 Nov 2020 15:09:24 +0530 Subject: soc: qcom: llcc-qcom: Add support for SM8250 SoC SM8250 SoC uses LLCC IP version 2. In this version, the WRSC_EN register needs to be written to enable the Write Sub Cache for each SCID. Hence, use a dedicated "write_scid_en" member with predefined values and write them for LLCC IP version 2. Reviewed-by: Sai Prakash Ranjan Signed-off-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/20201130093924.45057-5-manivannan.sadhasivam@linaro.org Signed-off-by: Bjorn Andersson --- drivers/soc/qcom/llcc-qcom.c | 38 ++++++++++++++++++++++++++++++++++++++ include/linux/soc/qcom/llcc-qcom.h | 1 + 2 files changed, 39 insertions(+) (limited to 'include/linux') diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c index a559617ea7c0..8403a77b59fe 100644 --- a/drivers/soc/qcom/llcc-qcom.c +++ b/drivers/soc/qcom/llcc-qcom.c @@ -51,6 +51,7 @@ #define LLCC_TRP_SCID_DIS_CAP_ALLOC 0x21f00 #define LLCC_TRP_PCB_ACT 0x21f04 +#define LLCC_TRP_WRSC_EN 0x21f20 #define BANK_OFFSET_STRIDE 0x80000 @@ -77,6 +78,7 @@ * then the ways assigned to this client are not flushed on power * collapse. * @activate_on_init: Activate the slice immediately after it is programmed + * @write_scid_en: Bit enables write cache support for a given scid. */ struct llcc_slice_config { u32 usecase_id; @@ -91,6 +93,7 @@ struct llcc_slice_config { bool dis_cap_alloc; bool retain_on_pc; bool activate_on_init; + bool write_scid_en; }; struct qcom_llcc_config { @@ -151,6 +154,25 @@ static const struct llcc_slice_config sm8150_data[] = { { LLCC_WRCACHE, 31, 128, 1, 1, 0xFFF, 0x0, 0, 0, 0, 0, 0 }, }; +static const struct llcc_slice_config sm8250_data[] = { + { LLCC_CPUSS, 1, 3072, 1, 1, 0xfff, 0x0, 0, 0, 0, 1, 1, 0 }, + { LLCC_VIDSC0, 2, 512, 3, 1, 0xfff, 0x0, 0, 0, 0, 1, 0, 0 }, + { LLCC_AUDIO, 6, 1024, 1, 0, 0xfff, 0x0, 0, 0, 0, 0, 0, 0 }, + { LLCC_CMPT, 10, 1024, 1, 0, 0xfff, 0x0, 0, 0, 0, 0, 0, 0 }, + { LLCC_GPUHTW, 11, 1024, 1, 1, 0xfff, 0x0, 0, 0, 0, 1, 0, 0 }, + { LLCC_GPU, 12, 1024, 1, 0, 0xfff, 0x0, 0, 0, 0, 1, 0, 1 }, + { LLCC_MMUHWT, 13, 1024, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, + { LLCC_CMPTDMA, 15, 1024, 1, 0, 0xfff, 0x0, 0, 0, 0, 1, 0, 0 }, + { LLCC_DISP, 16, 3072, 1, 1, 0xfff, 0x0, 0, 0, 0, 1, 0, 0 }, + { LLCC_VIDFW, 17, 512, 1, 0, 0xfff, 0x0, 0, 0, 0, 1, 0, 0 }, + { LLCC_AUDHW, 22, 1024, 1, 1, 0xfff, 0x0, 0, 0, 0, 1, 0, 0 }, + { LLCC_NPU, 23, 3072, 1, 1, 0xfff, 0x0, 0, 0, 0, 1, 0, 0 }, + { LLCC_WLHW, 24, 1024, 1, 0, 0xfff, 0x0, 0, 0, 0, 1, 0, 0 }, + { LLCC_CVP, 28, 256, 3, 1, 0xfff, 0x0, 0, 0, 0, 1, 0, 0 }, + { LLCC_APTCM, 30, 128, 3, 0, 0x0, 0x3, 1, 0, 0, 1, 0, 0 }, + { LLCC_WRCACHE, 31, 256, 1, 1, 0xfff, 0x0, 0, 0, 0, 0, 1, 0 }, +}; + static const struct qcom_llcc_config sc7180_cfg = { .sct_data = sc7180_data, .size = ARRAY_SIZE(sc7180_data), @@ -168,6 +190,11 @@ static const struct qcom_llcc_config sm8150_cfg = { .size = ARRAY_SIZE(sm8150_data), }; +static const struct qcom_llcc_config sm8250_cfg = { + .sct_data = sm8250_data, + .size = ARRAY_SIZE(sm8250_data), +}; + static struct llcc_drv_data *drv_data = (void *) -EPROBE_DEFER; /** @@ -417,6 +444,16 @@ static int _qcom_llcc_cfg_program(const struct llcc_slice_config *config, return ret; } + if (drv_data->major_version == 2) { + u32 wren; + + wren = config->write_scid_en << config->slice_id; + ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_WRSC_EN, + BIT(config->slice_id), wren); + if (ret) + return ret; + } + if (config->activate_on_init) { desc.slice_id = config->slice_id; ret = llcc_slice_activate(&desc); @@ -571,6 +608,7 @@ static const struct of_device_id qcom_llcc_of_match[] = { { .compatible = "qcom,sc7180-llcc", .data = &sc7180_cfg }, { .compatible = "qcom,sdm845-llcc", .data = &sdm845_cfg }, { .compatible = "qcom,sm8150-llcc", .data = &sm8150_cfg }, + { .compatible = "qcom,sm8250-llcc", .data = &sm8250_cfg }, { } }; diff --git a/include/linux/soc/qcom/llcc-qcom.h b/include/linux/soc/qcom/llcc-qcom.h index d17a3de80510..64fc582ae415 100644 --- a/include/linux/soc/qcom/llcc-qcom.h +++ b/include/linux/soc/qcom/llcc-qcom.h @@ -29,6 +29,7 @@ #define LLCC_AUDHW 22 #define LLCC_NPU 23 #define LLCC_WLHW 24 +#define LLCC_CVP 28 #define LLCC_MODPE 29 #define LLCC_APTCM 30 #define LLCC_WRCACHE 31 -- cgit v1.2.3 From a2ff95e018f1d2bc816f3078d5110a655e355f18 Mon Sep 17 00:00:00 2001 From: Mark Pearson Date: Tue, 29 Dec 2020 19:18:26 -0500 Subject: ACPI: platform: Add platform profile support This is the initial implementation of the platform-profile feature. It provides the details discussed and outlined in the sysfs-platform_profile document. Many modern systems have the ability to modify the operating profile to control aspects like fan speed, temperature and power levels. This module provides a common sysfs interface that platform modules can register against to control their individual profile options. Signed-off-by: Mark Pearson Reviewed-by: Hans de Goede [ rjw: Use full words in enum values names ] Signed-off-by: Rafael J. Wysocki --- drivers/acpi/Kconfig | 17 ++++ drivers/acpi/Makefile | 1 + drivers/acpi/platform_profile.c | 181 +++++++++++++++++++++++++++++++++++++++ include/linux/platform_profile.h | 39 +++++++++ 4 files changed, 238 insertions(+) create mode 100644 drivers/acpi/platform_profile.c create mode 100644 include/linux/platform_profile.h (limited to 'include/linux') diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index edf1558c1105..5ddff93e38c2 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -326,6 +326,23 @@ config ACPI_THERMAL To compile this driver as a module, choose M here: the module will be called thermal. +config ACPI_PLATFORM_PROFILE + tristate "ACPI Platform Profile Driver" + default m + help + This driver adds support for platform-profiles on platforms that + support it. + + Platform-profiles can be used to control the platform behaviour. For + example whether to operate in a lower power mode, in a higher + power performance mode or between the two. + + This driver provides the sysfs interface and is used as the registration + point for platform specific drivers. + + Which profiles are supported is determined on a per-platform basis and + should be obtained from the platform specific driver. + config ACPI_CUSTOM_DSDT_FILE string "Custom DSDT Table file to include" default "" diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile index 076894a3330f..52b627c7f977 100644 --- a/drivers/acpi/Makefile +++ b/drivers/acpi/Makefile @@ -79,6 +79,7 @@ obj-$(CONFIG_ACPI_PCI_SLOT) += pci_slot.o obj-$(CONFIG_ACPI_PROCESSOR) += processor.o obj-$(CONFIG_ACPI) += container.o obj-$(CONFIG_ACPI_THERMAL) += thermal.o +obj-$(CONFIG_ACPI_PLATFORM_PROFILE) += platform_profile.o obj-$(CONFIG_ACPI_NFIT) += nfit/ obj-$(CONFIG_ACPI_NUMA) += numa/ obj-$(CONFIG_ACPI) += acpi_memhotplug.o diff --git a/drivers/acpi/platform_profile.c b/drivers/acpi/platform_profile.c new file mode 100644 index 000000000000..91be50a32cc8 --- /dev/null +++ b/drivers/acpi/platform_profile.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +/* Platform profile sysfs interface */ + +#include +#include +#include +#include +#include +#include + +static const struct platform_profile_handler *cur_profile; +static DEFINE_MUTEX(profile_lock); + +static const char * const profile_names[] = { + [PLATFORM_PROFILE_LOW_POWER] = "low-power", + [PLATFORM_PROFILE_COOL] = "cool", + [PLATFORM_PROFILE_QUIET] = "quiet", + [PLATFORM_PROFILE_BALANCED] = "balanced", + [PLATFORM_PROFILE_PERFORMANCE] = "performance", +}; +static_assert(ARRAY_SIZE(profile_names) == PLATFORM_PROFILE_LAST); + +static ssize_t platform_profile_choices_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int len = 0; + int err, i; + + err = mutex_lock_interruptible(&profile_lock); + if (err) + return err; + + if (!cur_profile) { + mutex_unlock(&profile_lock); + return -ENODEV; + } + + for_each_set_bit(i, cur_profile->choices, PLATFORM_PROFILE_LAST) { + if (len == 0) + len += sysfs_emit_at(buf, len, "%s", profile_names[i]); + else + len += sysfs_emit_at(buf, len, " %s", profile_names[i]); + } + len += sysfs_emit_at(buf, len, "\n"); + mutex_unlock(&profile_lock); + return len; +} + +static ssize_t platform_profile_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + enum platform_profile_option profile = PLATFORM_PROFILE_BALANCED; + int err; + + err = mutex_lock_interruptible(&profile_lock); + if (err) + return err; + + if (!cur_profile) { + mutex_unlock(&profile_lock); + return -ENODEV; + } + + err = cur_profile->profile_get(&profile); + mutex_unlock(&profile_lock); + if (err) + return err; + + /* Check that profile is valid index */ + if (WARN_ON((profile < 0) || (profile >= ARRAY_SIZE(profile_names)))) + return -EIO; + + return sysfs_emit(buf, "%s\n", profile_names[profile]); +} + +static ssize_t platform_profile_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int err, i; + + err = mutex_lock_interruptible(&profile_lock); + if (err) + return err; + + if (!cur_profile) { + mutex_unlock(&profile_lock); + return -ENODEV; + } + + /* Scan for a matching profile */ + i = sysfs_match_string(profile_names, buf); + if (i < 0) { + mutex_unlock(&profile_lock); + return -EINVAL; + } + + /* Check that platform supports this profile choice */ + if (!test_bit(i, cur_profile->choices)) { + mutex_unlock(&profile_lock); + return -EOPNOTSUPP; + } + + err = cur_profile->profile_set(i); + mutex_unlock(&profile_lock); + if (err) + return err; + return count; +} + +static DEVICE_ATTR_RO(platform_profile_choices); +static DEVICE_ATTR_RW(platform_profile); + +static struct attribute *platform_profile_attrs[] = { + &dev_attr_platform_profile_choices.attr, + &dev_attr_platform_profile.attr, + NULL +}; + +static const struct attribute_group platform_profile_group = { + .attrs = platform_profile_attrs +}; + +void platform_profile_notify(void) +{ + if (!cur_profile) + return; + sysfs_notify(acpi_kobj, NULL, "platform_profile"); +} +EXPORT_SYMBOL_GPL(platform_profile_notify); + +int platform_profile_register(const struct platform_profile_handler *pprof) +{ + int err; + + mutex_lock(&profile_lock); + /* We can only have one active profile */ + if (cur_profile) { + mutex_unlock(&profile_lock); + return -EEXIST; + } + + /* Sanity check the profile handler field are set */ + if (!pprof || bitmap_empty(pprof->choices, PLATFORM_PROFILE_LAST) || + !pprof->profile_set || !pprof->profile_get) { + mutex_unlock(&profile_lock); + return -EINVAL; + } + + err = sysfs_create_group(acpi_kobj, &platform_profile_group); + if (err) { + mutex_unlock(&profile_lock); + return err; + } + + cur_profile = pprof; + mutex_unlock(&profile_lock); + return 0; +} +EXPORT_SYMBOL_GPL(platform_profile_register); + +int platform_profile_remove(void) +{ + mutex_lock(&profile_lock); + if (!cur_profile) { + mutex_unlock(&profile_lock); + return -ENODEV; + } + + sysfs_remove_group(acpi_kobj, &platform_profile_group); + cur_profile = NULL; + mutex_unlock(&profile_lock); + return 0; +} +EXPORT_SYMBOL_GPL(platform_profile_remove); + +MODULE_AUTHOR("Mark Pearson "); +MODULE_LICENSE("GPL"); diff --git a/include/linux/platform_profile.h b/include/linux/platform_profile.h new file mode 100644 index 000000000000..3623d7108421 --- /dev/null +++ b/include/linux/platform_profile.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Platform profile sysfs interface + * + * See Documentation/ABI/testing/sysfs-platform_profile.rst for more + * information. + */ + +#ifndef _PLATFORM_PROFILE_H_ +#define _PLATFORM_PROFILE_H_ + +#include + +/* + * If more options are added please update profile_names + * array in platform-profile.c and sysfs-platform-profile.rst + * documentation. + */ + +enum platform_profile_option { + PLATFORM_PROFILE_LOW_POWER, + PLATFORM_PROFILE_COOL, + PLATFORM_PROFILE_QUIET, + PLATFORM_PROFILE_BALANCED, + PLATFORM_PROFILE_PERFORMANCE, + PLATFORM_PROFILE_LAST, /*must always be last */ +}; + +struct platform_profile_handler { + unsigned long choices[BITS_TO_LONGS(PLATFORM_PROFILE_LAST)]; + int (*profile_get)(enum platform_profile_option *profile); + int (*profile_set)(enum platform_profile_option profile); +}; + +int platform_profile_register(const struct platform_profile_handler *pprof); +int platform_profile_remove(void); +void platform_profile_notify(void); + +#endif /*_PLATFORM_PROFILE_H_*/ -- cgit v1.2.3 From 8280de6ab07b4c63eb607662754f151e539031a1 Mon Sep 17 00:00:00 2001 From: Wesley Cheng Date: Tue, 29 Dec 2020 15:03:30 -0800 Subject: usb: gadget: composite: Split composite reset and disconnect Add a specific composite reset API to differentiate between disconnect and reset events. This is needed for adjusting the current draw accordingly based on the USB battery charging specification. The device is only allowed to draw the 500/900 mA (HS/SS) while in the CONFIGURED state, and only 100 mA in the connected and UNCONFIGURED state. Reviewed-by: Peter Chen Signed-off-by: Wesley Cheng Link: https://lore.kernel.org/r/1609283011-21997-3-git-send-email-wcheng@codeaurora.org Signed-off-by: Greg Kroah-Hartman --- drivers/usb/gadget/composite.c | 21 +++++++++++++++++++-- include/linux/usb/composite.h | 2 ++ 2 files changed, 21 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c index c6d455f2bb92..86e86a005e25 100644 --- a/drivers/usb/gadget/composite.c +++ b/drivers/usb/gadget/composite.c @@ -2036,7 +2036,7 @@ done: return value; } -void composite_disconnect(struct usb_gadget *gadget) +static void __composite_disconnect(struct usb_gadget *gadget) { struct usb_composite_dev *cdev = get_gadget_data(gadget); unsigned long flags; @@ -2053,6 +2053,23 @@ void composite_disconnect(struct usb_gadget *gadget) spin_unlock_irqrestore(&cdev->lock, flags); } +void composite_disconnect(struct usb_gadget *gadget) +{ + usb_gadget_vbus_draw(gadget, 0); + __composite_disconnect(gadget); +} + +void composite_reset(struct usb_gadget *gadget) +{ + /* + * Section 1.4.13 Standard Downstream Port of the USB battery charging + * specification v1.2 states that a device connected on a SDP shall only + * draw at max 100mA while in a connected, but unconfigured state. + */ + usb_gadget_vbus_draw(gadget, 100); + __composite_disconnect(gadget); +} + /*-------------------------------------------------------------------------*/ static ssize_t suspended_show(struct device *dev, struct device_attribute *attr, @@ -2373,7 +2390,7 @@ static const struct usb_gadget_driver composite_driver_template = { .unbind = composite_unbind, .setup = composite_setup, - .reset = composite_disconnect, + .reset = composite_reset, .disconnect = composite_disconnect, .suspend = composite_suspend, diff --git a/include/linux/usb/composite.h b/include/linux/usb/composite.h index a2d229ab63ba..5646dad886e6 100644 --- a/include/linux/usb/composite.h +++ b/include/linux/usb/composite.h @@ -525,6 +525,8 @@ extern struct usb_string *usb_gstrings_attach(struct usb_composite_dev *cdev, extern int usb_string_ids_n(struct usb_composite_dev *c, unsigned n); extern void composite_disconnect(struct usb_gadget *gadget); +extern void composite_reset(struct usb_gadget *gadget); + extern int composite_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl); extern void composite_suspend(struct usb_gadget *gadget); -- cgit v1.2.3 From 6c6ec2b0a3e0381d886d531bd1471dfdb1509237 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 17 Dec 2020 09:19:09 -0700 Subject: fs: add support for LOOKUP_CACHED io_uring always punts opens to async context, since there's no control over whether the lookup blocks or not. Add LOOKUP_CACHED to support just doing the fast RCU based lookups, which we know will not block. If we can do a cached path resolution of the filename, then we don't have to always punt lookups for a worker. During path resolution, we always do LOOKUP_RCU first. If that fails and we terminate LOOKUP_RCU, then fail a LOOKUP_CACHED attempt as well. Cc: Al Viro Signed-off-by: Jens Axboe Signed-off-by: Al Viro --- fs/namei.c | 9 +++++++++ include/linux/namei.h | 1 + 2 files changed, 10 insertions(+) (limited to 'include/linux') diff --git a/fs/namei.c b/fs/namei.c index 2ee219497460..4cae88733a5c 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -686,6 +686,8 @@ static bool try_to_unlazy(struct nameidata *nd) BUG_ON(!(nd->flags & LOOKUP_RCU)); nd->flags &= ~LOOKUP_RCU; + if (nd->flags & LOOKUP_CACHED) + goto out1; if (unlikely(!legitimize_links(nd))) goto out1; if (unlikely(!legitimize_path(nd, &nd->path, nd->seq))) @@ -722,6 +724,8 @@ static bool try_to_unlazy_next(struct nameidata *nd, struct dentry *dentry, unsi BUG_ON(!(nd->flags & LOOKUP_RCU)); nd->flags &= ~LOOKUP_RCU; + if (nd->flags & LOOKUP_CACHED) + goto out2; if (unlikely(!legitimize_links(nd))) goto out2; if (unlikely(!legitimize_mnt(nd->path.mnt, nd->m_seq))) @@ -792,6 +796,7 @@ static int complete_walk(struct nameidata *nd) */ if (!(nd->flags & (LOOKUP_ROOT | LOOKUP_IS_SCOPED))) nd->root.mnt = NULL; + nd->flags &= ~LOOKUP_CACHED; if (!try_to_unlazy(nd)) return -ECHILD; } @@ -2204,6 +2209,10 @@ static const char *path_init(struct nameidata *nd, unsigned flags) int error; const char *s = nd->name->name; + /* LOOKUP_CACHED requires RCU, ask caller to retry */ + if ((flags & (LOOKUP_RCU | LOOKUP_CACHED)) == LOOKUP_CACHED) + return ERR_PTR(-EAGAIN); + if (!*s) flags &= ~LOOKUP_RCU; if (flags & LOOKUP_RCU) diff --git a/include/linux/namei.h b/include/linux/namei.h index a4bb992623c4..b9605b2b46e7 100644 --- a/include/linux/namei.h +++ b/include/linux/namei.h @@ -46,6 +46,7 @@ enum {LAST_NORM, LAST_ROOT, LAST_DOT, LAST_DOTDOT}; #define LOOKUP_NO_XDEV 0x040000 /* No mountpoint crossing. */ #define LOOKUP_BENEATH 0x080000 /* No escaping from starting point. */ #define LOOKUP_IN_ROOT 0x100000 /* Treat dirfd as fs root. */ +#define LOOKUP_CACHED 0x200000 /* Only do cached lookup */ /* LOOKUP_* flags which do scope-related checks based on the dirfd. */ #define LOOKUP_IS_SCOPED (LOOKUP_BENEATH | LOOKUP_IN_ROOT) -- cgit v1.2.3 From 99668f618062816ca7ba639b007eb145b9d3d41e Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 17 Dec 2020 09:19:10 -0700 Subject: fs: expose LOOKUP_CACHED through openat2() RESOLVE_CACHED Now that we support non-blocking path resolution internally, expose it via openat2() in the struct open_how ->resolve flags. This allows applications using openat2() to limit path resolution to the extent that it is already cached. If the lookup cannot be satisfied in a non-blocking manner, openat2(2) will return -1/-EAGAIN. Cc: Al Viro Signed-off-by: Jens Axboe Signed-off-by: Al Viro --- fs/open.c | 6 ++++++ include/linux/fcntl.h | 2 +- include/uapi/linux/openat2.h | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/fs/open.c b/fs/open.c index 1e06e443a565..ca5444733acd 100644 --- a/fs/open.c +++ b/fs/open.c @@ -1091,6 +1091,12 @@ inline int build_open_flags(const struct open_how *how, struct open_flags *op) lookup_flags |= LOOKUP_BENEATH; if (how->resolve & RESOLVE_IN_ROOT) lookup_flags |= LOOKUP_IN_ROOT; + if (how->resolve & RESOLVE_CACHED) { + /* Don't bother even trying for create/truncate/tmpfile open */ + if (flags & (O_TRUNC | O_CREAT | O_TMPFILE)) + return -EAGAIN; + lookup_flags |= LOOKUP_CACHED; + } op->lookup_flags = lookup_flags; return 0; diff --git a/include/linux/fcntl.h b/include/linux/fcntl.h index 921e750843e6..766fcd973beb 100644 --- a/include/linux/fcntl.h +++ b/include/linux/fcntl.h @@ -19,7 +19,7 @@ /* List of all valid flags for the how->resolve argument: */ #define VALID_RESOLVE_FLAGS \ (RESOLVE_NO_XDEV | RESOLVE_NO_MAGICLINKS | RESOLVE_NO_SYMLINKS | \ - RESOLVE_BENEATH | RESOLVE_IN_ROOT) + RESOLVE_BENEATH | RESOLVE_IN_ROOT | RESOLVE_CACHED) /* List of all open_how "versions". */ #define OPEN_HOW_SIZE_VER0 24 /* sizeof first published struct */ diff --git a/include/uapi/linux/openat2.h b/include/uapi/linux/openat2.h index 58b1eb711360..a5feb7604948 100644 --- a/include/uapi/linux/openat2.h +++ b/include/uapi/linux/openat2.h @@ -35,5 +35,9 @@ struct open_how { #define RESOLVE_IN_ROOT 0x10 /* Make all jumps to "/" and ".." be scoped inside the dirfd (similar to chroot(2)). */ +#define RESOLVE_CACHED 0x20 /* Only complete if resolution can be + completed through cached lookup. May + return -EAGAIN if that's not + possible. */ #endif /* _UAPI_LINUX_OPENAT2_H */ -- cgit v1.2.3 From 4704bd317108c94b6e2d8309f3dbb70d2015568a Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 16 Nov 2020 11:18:16 +0100 Subject: list: Fix a typo at the kernel-doc markup hlist_add_behing -> hlist_add_behind Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Paul E. McKenney --- include/linux/list.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/list.h b/include/linux/list.h index 89bdc92e75c3..f2af4b4aa4e9 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -901,7 +901,7 @@ static inline void hlist_add_before(struct hlist_node *n, } /** - * hlist_add_behing - add a new entry after the one specified + * hlist_add_behind - add a new entry after the one specified * @n: new entry to be added * @prev: hlist node to add it after, which must be non-NULL */ -- cgit v1.2.3 From 5130b8fd06901c1b3a4bd0d0f5c5ea99b2b0a6f0 Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Fri, 20 Nov 2020 12:49:16 +0100 Subject: rcu: Introduce kfree_rcu() single-argument macro There is a kvfree_rcu() single argument macro that handles pointers returned by kvmalloc(). Even though it also handles pointer returned by kmalloc(), readability suffers. This commit therefore updates the kfree_rcu() macro to explicitly pair with kmalloc(), thus improving readability. Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index de0826411311..b95373ea3b02 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -851,8 +851,9 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) /** * kfree_rcu() - kfree an object after a grace period. - * @ptr: pointer to kfree - * @rhf: the name of the struct rcu_head within the type of @ptr. + * @ptr: pointer to kfree for both single- and double-argument invocations. + * @rhf: the name of the struct rcu_head within the type of @ptr, + * but only for double-argument invocations. * * Many rcu callbacks functions just call kfree() on the base structure. * These functions are trivial, but their size adds up, and furthermore @@ -875,13 +876,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) * The BUILD_BUG_ON check must not involve any function calls, hence the * checks are done in macros here. */ -#define kfree_rcu(ptr, rhf) \ -do { \ - typeof (ptr) ___p = (ptr); \ - \ - if (___p) \ - __kvfree_rcu(&((___p)->rhf), offsetof(typeof(*(ptr)), rhf)); \ -} while (0) +#define kfree_rcu kvfree_rcu /** * kvfree_rcu() - kvfree an object after a grace period. @@ -913,7 +908,14 @@ do { \ kvfree_rcu_arg_2, kvfree_rcu_arg_1)(__VA_ARGS__) #define KVFREE_GET_MACRO(_1, _2, NAME, ...) NAME -#define kvfree_rcu_arg_2(ptr, rhf) kfree_rcu(ptr, rhf) +#define kvfree_rcu_arg_2(ptr, rhf) \ +do { \ + typeof (ptr) ___p = (ptr); \ + \ + if (___p) \ + __kvfree_rcu(&((___p)->rhf), offsetof(typeof(*(ptr)), rhf)); \ +} while (0) + #define kvfree_rcu_arg_1(ptr) \ do { \ typeof(ptr) ___p = (ptr); \ -- cgit v1.2.3 From 5ea5d1ed572cb5ac173674fe770252253d2d9e27 Mon Sep 17 00:00:00 2001 From: "Uladzislau Rezki (Sony)" Date: Fri, 20 Nov 2020 12:49:17 +0100 Subject: rcu: Eliminate the __kvfree_rcu() macro This commit open-codes the __kvfree_rcu() macro, thus saving a few lines of code and improving readability. Signed-off-by: Uladzislau Rezki (Sony) Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index b95373ea3b02..f1576cde5951 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -840,15 +840,6 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) */ #define __is_kvfree_rcu_offset(offset) ((offset) < 4096) -/* - * Helper macro for kfree_rcu() to prevent argument-expansion eyestrain. - */ -#define __kvfree_rcu(head, offset) \ - do { \ - BUILD_BUG_ON(!__is_kvfree_rcu_offset(offset)); \ - kvfree_call_rcu(head, (rcu_callback_t)(unsigned long)(offset)); \ - } while (0) - /** * kfree_rcu() - kfree an object after a grace period. * @ptr: pointer to kfree for both single- and double-argument invocations. @@ -866,7 +857,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) * Because the functions are not allowed in the low-order 4096 bytes of * kernel virtual memory, offsets up to 4095 bytes can be accommodated. * If the offset is larger than 4095 bytes, a compile-time error will - * be generated in __kvfree_rcu(). If this error is triggered, you can + * be generated in kvfree_rcu_arg_2(). If this error is triggered, you can * either fall back to use of call_rcu() or rearrange the structure to * position the rcu_head structure into the first 4096 bytes. * @@ -912,8 +903,11 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) do { \ typeof (ptr) ___p = (ptr); \ \ - if (___p) \ - __kvfree_rcu(&((___p)->rhf), offsetof(typeof(*(ptr)), rhf)); \ + if (___p) { \ + BUILD_BUG_ON(!__is_kvfree_rcu_offset(offsetof(typeof(*(ptr)), rhf))); \ + kvfree_call_rcu(&((___p)->rhf), (rcu_callback_t)(unsigned long) \ + (offsetof(typeof(*(ptr)), rhf))); \ + } \ } while (0) #define kvfree_rcu_arg_1(ptr) \ -- cgit v1.2.3 From 74612a07b83fc46c2b2e6f71a541d55b024ebefc Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 12 Nov 2020 16:34:09 -0800 Subject: srcu: Make Tiny SRCU use multi-bit grace-period counter There is a need for a polling interface for SRCU grace periods. This polling needs to distinguish between an SRCU instance being idle on the one hand or in the middle of a grace period on the other. This commit therefore converts the Tiny SRCU srcu_struct structure's srcu_idx from a defacto boolean to a free-running counter, using the bottom bit to indicate that a grace period is in progress. The second-from-bottom bit is thus used as the index returned by srcu_read_lock(). Link: https://lore.kernel.org/rcu/20201112201547.GF3365678@moria.home.lan/ Reported-by: Kent Overstreet [ paulmck: Fix ->srcu_lock_nesting[] indexing per Neeraj Upadhyay. ] Reviewed-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- include/linux/srcutiny.h | 6 +++--- kernel/rcu/srcutiny.c | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index 5a5a1941ca15..b8b42d0a24f1 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -15,7 +15,7 @@ struct srcu_struct { short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */ - short srcu_idx; /* Current reader array element. */ + unsigned short srcu_idx; /* Current reader array element in bit 0x2. */ u8 srcu_gp_running; /* GP workqueue running? */ u8 srcu_gp_waiting; /* GP waiting for readers? */ struct swait_queue_head srcu_wq; @@ -59,7 +59,7 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp) { int idx; - idx = READ_ONCE(ssp->srcu_idx); + idx = ((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1; WRITE_ONCE(ssp->srcu_lock_nesting[idx], ssp->srcu_lock_nesting[idx] + 1); return idx; } @@ -80,7 +80,7 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, { int idx; - idx = READ_ONCE(ssp->srcu_idx) & 0x1; + idx = ((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1; pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n", tt, tf, idx, READ_ONCE(ssp->srcu_lock_nesting[!idx]), diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 6208c1dae5c9..5598cf6f1edc 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -124,11 +124,12 @@ void srcu_drive_gp(struct work_struct *wp) ssp->srcu_cb_head = NULL; ssp->srcu_cb_tail = &ssp->srcu_cb_head; local_irq_enable(); - idx = ssp->srcu_idx; - WRITE_ONCE(ssp->srcu_idx, !ssp->srcu_idx); + idx = (ssp->srcu_idx & 0x2) / 2; + WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */ swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx])); WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */ + WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); /* Invoke the callbacks we removed above. */ while (lh) { -- cgit v1.2.3 From 8b5bd67cf6422b63ee100d76d8de8960ca2df7f0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 13 Nov 2020 12:54:48 -0800 Subject: srcu: Provide polling interfaces for Tiny SRCU grace periods There is a need for a polling interface for SRCU grace periods, so this commit supplies get_state_synchronize_srcu(), start_poll_synchronize_srcu(), and poll_state_synchronize_srcu() for this purpose. The first can be used if future grace periods are inevitable (perhaps due to a later call_srcu() invocation), the second if future grace periods might not otherwise happen, and the third to check if a grace period has elapsed since the corresponding call to either of the first two. As with get_state_synchronize_rcu() and cond_synchronize_rcu(), the return value from either get_state_synchronize_srcu() or start_poll_synchronize_srcu() must be passed in to a later call to poll_state_synchronize_srcu(). Link: https://lore.kernel.org/rcu/20201112201547.GF3365678@moria.home.lan/ Reported-by: Kent Overstreet [ paulmck: Add EXPORT_SYMBOL_GPL() per kernel test robot feedback. ] [ paulmck: Apply feedback from Neeraj Upadhyay. ] Link: https://lore.kernel.org/lkml/20201117004017.GA7444@paulmck-ThinkPad-P72/ Reviewed-by: Neeraj Upadhyay Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 2 ++ include/linux/srcu.h | 3 +++ include/linux/srcutiny.h | 1 + kernel/rcu/srcutiny.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 59 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index de0826411311..e09c0d87b3c3 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -33,6 +33,8 @@ #define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b)) #define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b)) #define ulong2long(a) (*(long *)(&(a))) +#define USHORT_CMP_GE(a, b) (USHRT_MAX / 2 >= (unsigned short)((a) - (b))) +#define USHORT_CMP_LT(a, b) (USHRT_MAX / 2 < (unsigned short)((a) - (b))) /* Exported common interfaces */ void call_rcu(struct rcu_head *head, rcu_callback_t func); diff --git a/include/linux/srcu.h b/include/linux/srcu.h index e432cc92c73d..a0895bbf71ce 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -60,6 +60,9 @@ void cleanup_srcu_struct(struct srcu_struct *ssp); int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp); void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp); void synchronize_srcu(struct srcu_struct *ssp); +unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp); +unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp); +bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie); #ifdef CONFIG_DEBUG_LOCK_ALLOC diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index b8b42d0a24f1..0e0cf4d6a72a 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -16,6 +16,7 @@ struct srcu_struct { short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */ unsigned short srcu_idx; /* Current reader array element in bit 0x2. */ + unsigned short srcu_idx_max; /* Furthest future srcu_idx request. */ u8 srcu_gp_running; /* GP workqueue running? */ u8 srcu_gp_waiting; /* GP waiting for readers? */ struct swait_queue_head srcu_wq; diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 3bac1db85a85..26344dc6483b 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -34,6 +34,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) ssp->srcu_gp_running = false; ssp->srcu_gp_waiting = false; ssp->srcu_idx = 0; + ssp->srcu_idx_max = 0; INIT_WORK(&ssp->srcu_work, srcu_drive_gp); INIT_LIST_HEAD(&ssp->srcu_work.entry); return 0; @@ -84,6 +85,8 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) WARN_ON(ssp->srcu_gp_waiting); WARN_ON(ssp->srcu_cb_head); WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail); + WARN_ON(ssp->srcu_idx != ssp->srcu_idx_max); + WARN_ON(ssp->srcu_idx & 0x1); } EXPORT_SYMBOL_GPL(cleanup_srcu_struct); @@ -114,7 +117,7 @@ void srcu_drive_gp(struct work_struct *wp) struct srcu_struct *ssp; ssp = container_of(wp, struct srcu_struct, srcu_work); - if (ssp->srcu_gp_running || !READ_ONCE(ssp->srcu_cb_head)) + if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) return; /* Already running or nothing to do. */ /* Remove recently arrived callbacks and wait for readers. */ @@ -147,13 +150,19 @@ void srcu_drive_gp(struct work_struct *wp) * straighten that out. */ WRITE_ONCE(ssp->srcu_gp_running, false); - if (READ_ONCE(ssp->srcu_cb_head)) + if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) schedule_work(&ssp->srcu_work); } EXPORT_SYMBOL_GPL(srcu_drive_gp); static void srcu_gp_start_if_needed(struct srcu_struct *ssp) { + unsigned short cookie; + + cookie = get_state_synchronize_srcu(ssp); + if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) + return; + WRITE_ONCE(ssp->srcu_idx_max, cookie); if (!READ_ONCE(ssp->srcu_gp_running)) { if (likely(srcu_init_done)) schedule_work(&ssp->srcu_work); @@ -196,6 +205,48 @@ void synchronize_srcu(struct srcu_struct *ssp) } EXPORT_SYMBOL_GPL(synchronize_srcu); +/* + * get_state_synchronize_srcu - Provide an end-of-grace-period cookie + */ +unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp) +{ + unsigned long ret; + + barrier(); + ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1; + barrier(); + return ret & USHRT_MAX; +} +EXPORT_SYMBOL_GPL(get_state_synchronize_srcu); + +/* + * start_poll_synchronize_srcu - Provide cookie and start grace period + * + * The difference between this and get_state_synchronize_srcu() is that + * this function ensures that the poll_state_synchronize_srcu() will + * eventually return the value true. + */ +unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) +{ + unsigned long ret = get_state_synchronize_srcu(ssp); + + srcu_gp_start_if_needed(ssp); + return ret; +} +EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu); + +/* + * poll_state_synchronize_srcu - Has cookie's grace period ended? + */ +bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) +{ + bool ret = USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx), cookie); + + barrier(); + return ret; +} +EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu); + /* Lockdep diagnostics. */ void __init rcu_scheduler_starting(void) { -- cgit v1.2.3 From 281462e593483350d8072a118c6e072c550a80fa Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Mon, 28 Dec 2020 18:49:16 +0300 Subject: memory: tegra124-emc: Make driver modular Add modularization support to the Tegra124 EMC driver, which now can be compiled as a loadable kernel module. Note that EMC clock must be registered at clk-init time, otherwise PLLM will be disabled as unused clock at boot time if EMC driver is compiled as a module. Hence add a prepare/complete callbacks. similarly to what is done for the Tegra20/30 EMC drivers. Tested-by: Nicolas Chauvet Signed-off-by: Dmitry Osipenko Link: https://lore.kernel.org/r/20201228154920.18846-2-digetx@gmail.com Signed-off-by: Krzysztof Kozlowski --- drivers/clk/tegra/Kconfig | 3 +++ drivers/clk/tegra/Makefile | 2 +- drivers/clk/tegra/clk-tegra124-emc.c | 41 +++++++++++++++++++++++++++++++----- drivers/clk/tegra/clk-tegra124.c | 26 ++++++++++++++++++++--- drivers/clk/tegra/clk.h | 18 ++++++++++------ drivers/memory/tegra/Kconfig | 3 ++- drivers/memory/tegra/tegra124-emc.c | 31 ++++++++++++++++++--------- include/linux/clk/tegra.h | 8 +++++++ include/soc/tegra/emc.h | 16 -------------- 9 files changed, 106 insertions(+), 42 deletions(-) delete mode 100644 include/soc/tegra/emc.h (limited to 'include/linux') diff --git a/drivers/clk/tegra/Kconfig b/drivers/clk/tegra/Kconfig index deaa4605824c..90df619dc087 100644 --- a/drivers/clk/tegra/Kconfig +++ b/drivers/clk/tegra/Kconfig @@ -7,3 +7,6 @@ config TEGRA_CLK_DFLL depends on ARCH_TEGRA_124_SOC || ARCH_TEGRA_210_SOC select PM_OPP def_bool y + +config TEGRA124_CLK_EMC + bool diff --git a/drivers/clk/tegra/Makefile b/drivers/clk/tegra/Makefile index eec2313fd37e..7b1816856eb5 100644 --- a/drivers/clk/tegra/Makefile +++ b/drivers/clk/tegra/Makefile @@ -22,7 +22,7 @@ obj-$(CONFIG_ARCH_TEGRA_3x_SOC) += clk-tegra20-emc.o obj-$(CONFIG_ARCH_TEGRA_114_SOC) += clk-tegra114.o obj-$(CONFIG_ARCH_TEGRA_124_SOC) += clk-tegra124.o obj-$(CONFIG_TEGRA_CLK_DFLL) += clk-tegra124-dfll-fcpu.o -obj-$(CONFIG_TEGRA124_EMC) += clk-tegra124-emc.o +obj-$(CONFIG_TEGRA124_CLK_EMC) += clk-tegra124-emc.o obj-$(CONFIG_ARCH_TEGRA_132_SOC) += clk-tegra124.o obj-y += cvb.o obj-$(CONFIG_ARCH_TEGRA_210_SOC) += clk-tegra210.o diff --git a/drivers/clk/tegra/clk-tegra124-emc.c b/drivers/clk/tegra/clk-tegra124-emc.c index 745f9faa98d8..bdf6f4a51617 100644 --- a/drivers/clk/tegra/clk-tegra124-emc.c +++ b/drivers/clk/tegra/clk-tegra124-emc.c @@ -11,7 +11,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -21,7 +23,6 @@ #include #include -#include #include "clk.h" @@ -80,6 +81,9 @@ struct tegra_clk_emc { int num_timings; struct emc_timing *timings; spinlock_t *lock; + + tegra124_emc_prepare_timing_change_cb *prepare_timing_change; + tegra124_emc_complete_timing_change_cb *complete_timing_change; }; /* Common clock framework callback implementations */ @@ -176,6 +180,9 @@ static struct tegra_emc *emc_ensure_emc_driver(struct tegra_clk_emc *tegra) if (tegra->emc) return tegra->emc; + if (!tegra->prepare_timing_change || !tegra->complete_timing_change) + return NULL; + if (!tegra->emc_node) return NULL; @@ -241,7 +248,7 @@ static int emc_set_timing(struct tegra_clk_emc *tegra, div = timing->parent_rate / (timing->rate / 2) - 2; - err = tegra_emc_prepare_timing_change(emc, timing->rate); + err = tegra->prepare_timing_change(emc, timing->rate); if (err) return err; @@ -259,7 +266,7 @@ static int emc_set_timing(struct tegra_clk_emc *tegra, spin_unlock_irqrestore(tegra->lock, flags); - tegra_emc_complete_timing_change(emc, timing->rate); + tegra->complete_timing_change(emc, timing->rate); clk_hw_reparent(&tegra->hw, __clk_get_hw(timing->parent)); clk_disable_unprepare(tegra->prev_parent); @@ -473,8 +480,8 @@ static const struct clk_ops tegra_clk_emc_ops = { .get_parent = emc_get_parent, }; -struct clk *tegra_clk_register_emc(void __iomem *base, struct device_node *np, - spinlock_t *lock) +struct clk *tegra124_clk_register_emc(void __iomem *base, struct device_node *np, + spinlock_t *lock) { struct tegra_clk_emc *tegra; struct clk_init_data init; @@ -538,3 +545,27 @@ struct clk *tegra_clk_register_emc(void __iomem *base, struct device_node *np, return clk; }; + +void tegra124_clk_set_emc_callbacks(tegra124_emc_prepare_timing_change_cb *prep_cb, + tegra124_emc_complete_timing_change_cb *complete_cb) +{ + struct clk *clk = __clk_lookup("emc"); + struct tegra_clk_emc *tegra; + struct clk_hw *hw; + + if (clk) { + hw = __clk_get_hw(clk); + tegra = container_of(hw, struct tegra_clk_emc, hw); + + tegra->prepare_timing_change = prep_cb; + tegra->complete_timing_change = complete_cb; + } +} +EXPORT_SYMBOL_GPL(tegra124_clk_set_emc_callbacks); + +bool tegra124_clk_emc_driver_available(struct clk_hw *hw) +{ + struct tegra_clk_emc *tegra = container_of(hw, struct tegra_clk_emc, hw); + + return tegra->prepare_timing_change && tegra->complete_timing_change; +} diff --git a/drivers/clk/tegra/clk-tegra124.c b/drivers/clk/tegra/clk-tegra124.c index e931319dcc9d..934520aab6e3 100644 --- a/drivers/clk/tegra/clk-tegra124.c +++ b/drivers/clk/tegra/clk-tegra124.c @@ -1500,6 +1500,26 @@ static void __init tegra124_132_clock_init_pre(struct device_node *np) writel(plld_base, clk_base + PLLD_BASE); } +static struct clk *tegra124_clk_src_onecell_get(struct of_phandle_args *clkspec, + void *data) +{ + struct clk_hw *hw; + struct clk *clk; + + clk = of_clk_src_onecell_get(clkspec, data); + if (IS_ERR(clk)) + return clk; + + hw = __clk_get_hw(clk); + + if (clkspec->args[0] == TEGRA124_CLK_EMC) { + if (!tegra124_clk_emc_driver_available(hw)) + return ERR_PTR(-EPROBE_DEFER); + } + + return clk; +} + /** * tegra124_132_clock_init_post - clock initialization postamble for T124/T132 * @np: struct device_node * of the DT node for the SoC CAR IP block @@ -1516,10 +1536,10 @@ static void __init tegra124_132_clock_init_post(struct device_node *np) &pll_x_params); tegra_init_special_resets(1, tegra124_reset_assert, tegra124_reset_deassert); - tegra_add_of_provider(np, of_clk_src_onecell_get); + tegra_add_of_provider(np, tegra124_clk_src_onecell_get); - clks[TEGRA124_CLK_EMC] = tegra_clk_register_emc(clk_base, np, - &emc_lock); + clks[TEGRA124_CLK_EMC] = tegra124_clk_register_emc(clk_base, np, + &emc_lock); tegra_register_devclks(devclks, ARRAY_SIZE(devclks)); diff --git a/drivers/clk/tegra/clk.h b/drivers/clk/tegra/clk.h index 6b565f6b5f66..c3e36b5dcc75 100644 --- a/drivers/clk/tegra/clk.h +++ b/drivers/clk/tegra/clk.h @@ -881,16 +881,22 @@ void tegra_super_clk_gen5_init(void __iomem *clk_base, void __iomem *pmc_base, struct tegra_clk *tegra_clks, struct tegra_clk_pll_params *pll_params); -#ifdef CONFIG_TEGRA124_EMC -struct clk *tegra_clk_register_emc(void __iomem *base, struct device_node *np, - spinlock_t *lock); +#ifdef CONFIG_TEGRA124_CLK_EMC +struct clk *tegra124_clk_register_emc(void __iomem *base, struct device_node *np, + spinlock_t *lock); +bool tegra124_clk_emc_driver_available(struct clk_hw *emc_hw); #else -static inline struct clk *tegra_clk_register_emc(void __iomem *base, - struct device_node *np, - spinlock_t *lock) +static inline struct clk * +tegra124_clk_register_emc(void __iomem *base, struct device_node *np, + spinlock_t *lock) { return NULL; } + +static inline bool tegra124_clk_emc_driver_available(struct clk_hw *emc_hw) +{ + return false; +} #endif void tegra114_clock_tune_cpu_trimmers_high(void); diff --git a/drivers/memory/tegra/Kconfig b/drivers/memory/tegra/Kconfig index ca7077a06f4c..f5b451403c58 100644 --- a/drivers/memory/tegra/Kconfig +++ b/drivers/memory/tegra/Kconfig @@ -32,9 +32,10 @@ config TEGRA30_EMC external memory. config TEGRA124_EMC - bool "NVIDIA Tegra124 External Memory Controller driver" + tristate "NVIDIA Tegra124 External Memory Controller driver" default y depends on TEGRA_MC && ARCH_TEGRA_124_SOC + select TEGRA124_CLK_EMC help This driver is for the External Memory Controller (EMC) found on Tegra124 chips. The EMC controls the external DRAM on the board. diff --git a/drivers/memory/tegra/tegra124-emc.c b/drivers/memory/tegra/tegra124-emc.c index ee8ee39e98ed..edfbf6d6d357 100644 --- a/drivers/memory/tegra/tegra124-emc.c +++ b/drivers/memory/tegra/tegra124-emc.c @@ -9,16 +9,17 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include #include -#include #include #include @@ -562,8 +563,8 @@ static struct emc_timing *tegra_emc_find_timing(struct tegra_emc *emc, return timing; } -int tegra_emc_prepare_timing_change(struct tegra_emc *emc, - unsigned long rate) +static int tegra_emc_prepare_timing_change(struct tegra_emc *emc, + unsigned long rate) { struct emc_timing *timing = tegra_emc_find_timing(emc, rate); struct emc_timing *last = &emc->last_timing; @@ -790,8 +791,8 @@ int tegra_emc_prepare_timing_change(struct tegra_emc *emc, return 0; } -void tegra_emc_complete_timing_change(struct tegra_emc *emc, - unsigned long rate) +static void tegra_emc_complete_timing_change(struct tegra_emc *emc, + unsigned long rate) { struct emc_timing *timing = tegra_emc_find_timing(emc, rate); struct emc_timing *last = &emc->last_timing; @@ -987,6 +988,7 @@ static const struct of_device_id tegra_emc_of_match[] = { { .compatible = "nvidia,tegra132-emc" }, {} }; +MODULE_DEVICE_TABLE(of, tegra_emc_of_match); static struct device_node * tegra_emc_find_node_by_ram_code(struct device_node *node, u32 ram_code) @@ -1226,9 +1228,19 @@ static int tegra_emc_probe(struct platform_device *pdev) platform_set_drvdata(pdev, emc); + tegra124_clk_set_emc_callbacks(tegra_emc_prepare_timing_change, + tegra_emc_complete_timing_change); + if (IS_ENABLED(CONFIG_DEBUG_FS)) emc_debugfs_init(&pdev->dev, emc); + /* + * Don't allow the kernel module to be unloaded. Unloading adds some + * extra complexity which doesn't really worth the effort in a case of + * this driver. + */ + try_module_get(THIS_MODULE); + return 0; }; @@ -1240,9 +1252,8 @@ static struct platform_driver tegra_emc_driver = { .suppress_bind_attrs = true, }, }; +module_platform_driver(tegra_emc_driver); -static int tegra_emc_init(void) -{ - return platform_driver_register(&tegra_emc_driver); -} -subsys_initcall(tegra_emc_init); +MODULE_AUTHOR("Mikko Perttunen "); +MODULE_DESCRIPTION("NVIDIA Tegra124 EMC driver"); +MODULE_LICENSE("GPL v2"); diff --git a/include/linux/clk/tegra.h b/include/linux/clk/tegra.h index 3f01d43f0598..eb016fc9cc0b 100644 --- a/include/linux/clk/tegra.h +++ b/include/linux/clk/tegra.h @@ -136,6 +136,7 @@ extern void tegra210_clk_emc_dll_update_setting(u32 emc_dll_src_value); extern void tegra210_clk_emc_update_setting(u32 emc_src_value); struct clk; +struct tegra_emc; typedef long (tegra20_clk_emc_round_cb)(unsigned long rate, unsigned long min_rate, @@ -146,6 +147,13 @@ void tegra20_clk_set_emc_round_callback(tegra20_clk_emc_round_cb *round_cb, void *cb_arg); int tegra20_clk_prepare_emc_mc_same_freq(struct clk *emc_clk, bool same); +typedef int (tegra124_emc_prepare_timing_change_cb)(struct tegra_emc *emc, + unsigned long rate); +typedef void (tegra124_emc_complete_timing_change_cb)(struct tegra_emc *emc, + unsigned long rate); +void tegra124_clk_set_emc_callbacks(tegra124_emc_prepare_timing_change_cb *prep_cb, + tegra124_emc_complete_timing_change_cb *complete_cb); + struct tegra210_clk_emc_config { unsigned long rate; bool same_freq; diff --git a/include/soc/tegra/emc.h b/include/soc/tegra/emc.h deleted file mode 100644 index 05199a97ccf4..000000000000 --- a/include/soc/tegra/emc.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (c) 2014 NVIDIA Corporation. All rights reserved. - */ - -#ifndef __SOC_TEGRA_EMC_H__ -#define __SOC_TEGRA_EMC_H__ - -struct tegra_emc; - -int tegra_emc_prepare_timing_change(struct tegra_emc *emc, - unsigned long rate); -void tegra_emc_complete_timing_change(struct tegra_emc *emc, - unsigned long rate); - -#endif /* __SOC_TEGRA_EMC_H__ */ -- cgit v1.2.3 From 8553a979fcd03448a4096c7d431b7ee1a52bfca3 Mon Sep 17 00:00:00 2001 From: Utkarsh Patel Date: Wed, 9 Dec 2020 22:09:03 -0800 Subject: platform/chrome: cros_ec_typec: Send mux configuration acknowledgment to EC In some corner cases downgrade of the superspeed typec device(e.g. Dell typec Dock, apple dongle) was seen because before the SOC mux configuration finishes, EC starts configuring the next mux state. With this change, once the SOC mux is configured, kernel will send an acknowledgment to EC via Host command EC_CMD_USB_PD_MUX_ACK [1]. After sending the host event EC will wait for the acknowledgment from kernel before starting the PD negotiation for the next mux state. This helps to have a framework to build better error handling along with the synchronization of timing sensitive mux states. This change also brings in corresponding EC header updates from the EC code base [1]. [1]: https://chromium.googlesource.com/chromiumos/platform/ec/+/refs/heads/master/include/ec_commands.h Signed-off-by: Utkarsh Patel Reviewed-by: Prashant Malani Signed-off-by: Benson Leung Link: https://lore.kernel.org/r/20201210060903.2205-3-utkarsh.h.patel@intel.com --- drivers/platform/chrome/cros_ec_typec.c | 16 ++++++++++++++++ include/linux/platform_data/cros_ec_commands.h | 17 +++++++++++++++++ 2 files changed, 33 insertions(+) (limited to 'include/linux') diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c index 6068433dd2e2..e724a5eaef1c 100644 --- a/drivers/platform/chrome/cros_ec_typec.c +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -81,6 +81,7 @@ struct cros_typec_data { struct notifier_block nb; struct work_struct port_work; bool typec_cmd_supported; + bool needs_mux_ack; }; static int cros_typec_parse_port_props(struct typec_capability *cap, @@ -531,6 +532,7 @@ static int cros_typec_configure_mux(struct cros_typec_data *typec, int port_num, struct ec_response_usb_pd_control_v2 *pd_ctrl) { struct cros_typec_port *port = typec->ports[port_num]; + struct ec_params_usb_pd_mux_ack mux_ack; enum typec_orientation orientation; int ret; @@ -570,6 +572,18 @@ static int cros_typec_configure_mux(struct cros_typec_data *typec, int port_num, mux_flags); } + if (!typec->needs_mux_ack) + return ret; + + /* Sending Acknowledgment to EC */ + mux_ack.port = port_num; + + if (cros_typec_ec_command(typec, 0, EC_CMD_USB_PD_MUX_ACK, &mux_ack, + sizeof(mux_ack), NULL, 0) < 0) + dev_warn(typec->dev, + "Failed to send Mux ACK to EC for port: %d\n", + port_num); + return ret; } @@ -1053,6 +1067,8 @@ static int cros_typec_probe(struct platform_device *pdev) typec->typec_cmd_supported = !!cros_typec_feature_supported(typec, EC_FEATURE_TYPEC_CMD); + typec->needs_mux_ack = !!cros_typec_feature_supported(typec, + EC_FEATURE_TYPEC_MUX_REQUIRE_AP_ACK); ret = cros_typec_ec_command(typec, 0, EC_CMD_USB_PD_PORTS, NULL, 0, &resp, sizeof(resp)); diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h index 86376779ab31..9787715540c7 100644 --- a/include/linux/platform_data/cros_ec_commands.h +++ b/include/linux/platform_data/cros_ec_commands.h @@ -1286,6 +1286,16 @@ enum ec_feature_code { EC_FEATURE_ISH = 40, /* New TCPMv2 TYPEC_ prefaced commands supported */ EC_FEATURE_TYPEC_CMD = 41, + /* + * The EC will wait for direction from the AP to enter Type-C alternate + * modes or USB4. + */ + EC_FEATURE_TYPEC_REQUIRE_AP_MODE_ENTRY = 42, + /* + * The EC will wait for an acknowledge from the AP after setting the + * mux. + */ + EC_FEATURE_TYPEC_MUX_REQUIRE_AP_ACK = 43, }; #define EC_FEATURE_MASK_0(event_code) BIT(event_code % 32) @@ -6054,6 +6064,13 @@ struct ec_params_charger_control { uint8_t allow_charging; } __ec_align_size1; +/* Get ACK from the USB-C SS muxes */ +#define EC_CMD_USB_PD_MUX_ACK 0x0603 + +struct ec_params_usb_pd_mux_ack { + uint8_t port; /* USB-C port number */ +} __ec_align1; + /*****************************************************************************/ /* * Reserve a range of host commands for board-specific, experimental, or -- cgit v1.2.3 From f2485a2dc9f0f30fbdd013ad5772975100c71360 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 13 Jun 2020 00:08:44 -0400 Subject: elf_prstatus: collect the common part (everything before pr_reg) into a struct Preparations to doing i386 compat elf_prstatus sanely - rather than duplicating the beginning of compat_elf_prstatus, take these fields into a separate structure (compat_elf_prstatus_common), so that it could be reused. Due to the incestous relationship between binfmt_elf.c and compat_binfmt_elf.c we need the same shape change done to native struct elf_prstatus, gathering the fields prior to pr_reg into a new structure (struct elf_prstatus_common). Fortunately, offset of pr_reg is always a multiple of 16 with no padding right before it, so it's possible to turn all the stuff prior to it into a single member without disturbing the layout. [build fix from Geert Uytterhoeven folded in] Signed-off-by: Al Viro --- arch/ia64/kernel/crash.c | 2 +- arch/mips/kernel/binfmt_elfn32.c | 7 ++++++- arch/mips/kernel/binfmt_elfo32.c | 6 +++++- arch/powerpc/platforms/powernv/opal-core.c | 6 +++--- arch/s390/kernel/crash_dump.c | 2 +- fs/binfmt_elf.c | 8 ++++---- fs/binfmt_elf_fdpic.c | 22 +++++----------------- fs/compat_binfmt_elf.c | 1 + include/linux/elfcore-compat.h | 7 ++++++- include/linux/elfcore.h | 7 ++++++- kernel/kexec_core.c | 2 +- 11 files changed, 39 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c index fec70d662d0c..4f47741005d2 100644 --- a/arch/ia64/kernel/crash.c +++ b/arch/ia64/kernel/crash.c @@ -43,7 +43,7 @@ crash_save_this_cpu(void) elf_greg_t *dst = (elf_greg_t *)&(prstatus->pr_reg); memset(prstatus, 0, sizeof(*prstatus)); - prstatus->pr_pid = current->pid; + prstatus->common.pr_pid = current->pid; ia64_dump_cpu_regs(dst); cfm = dst[43]; diff --git a/arch/mips/kernel/binfmt_elfn32.c b/arch/mips/kernel/binfmt_elfn32.c index 6ee3f7218c67..136dc0c9300d 100644 --- a/arch/mips/kernel/binfmt_elfn32.c +++ b/arch/mips/kernel/binfmt_elfn32.c @@ -44,7 +44,8 @@ typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; #include #define elf_prstatus elf_prstatus32 -struct elf_prstatus32 +#define elf_prstatus_common elf_prstatus32_common +struct elf_prstatus32_common { struct elf_siginfo pr_info; /* Info associated with signal */ short pr_cursig; /* Current signal */ @@ -58,6 +59,10 @@ struct elf_prstatus32 struct old_timeval32 pr_stime; /* System time */ struct old_timeval32 pr_cutime;/* Cumulative user time */ struct old_timeval32 pr_cstime;/* Cumulative system time */ +}; +struct elf_prstatus32 +{ + struct elf_prstatus32_common common: elf_gregset_t pr_reg; /* GP registers */ int pr_fpvalid; /* True if math co-processor being used. */ }; diff --git a/arch/mips/kernel/binfmt_elfo32.c b/arch/mips/kernel/binfmt_elfo32.c index 6dd103d3cebb..b1f4b8f1dee7 100644 --- a/arch/mips/kernel/binfmt_elfo32.c +++ b/arch/mips/kernel/binfmt_elfo32.c @@ -49,7 +49,7 @@ typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; #include #define elf_prstatus elf_prstatus32 -struct elf_prstatus32 +struct elf_prstatus32_common { struct elf_siginfo pr_info; /* Info associated with signal */ short pr_cursig; /* Current signal */ @@ -63,6 +63,10 @@ struct elf_prstatus32 struct old_timeval32 pr_stime; /* System time */ struct old_timeval32 pr_cutime;/* Cumulative user time */ struct old_timeval32 pr_cstime;/* Cumulative system time */ +}; +struct elf_prstatus32 +{ + struct elf_prstatus32_common common: elf_gregset_t pr_reg; /* GP registers */ int pr_fpvalid; /* True if math co-processor being used. */ }; diff --git a/arch/powerpc/platforms/powernv/opal-core.c b/arch/powerpc/platforms/powernv/opal-core.c index 23571f0b555a..0d9ba70f7251 100644 --- a/arch/powerpc/platforms/powernv/opal-core.c +++ b/arch/powerpc/platforms/powernv/opal-core.c @@ -119,8 +119,8 @@ static void fill_prstatus(struct elf_prstatus *prstatus, int pir, * As a PIR value could also be '0', add an offset of '100' * to every PIR to avoid misinterpretations in GDB. */ - prstatus->pr_pid = cpu_to_be32(100 + pir); - prstatus->pr_ppid = cpu_to_be32(1); + prstatus->common.pr_pid = cpu_to_be32(100 + pir); + prstatus->common.pr_ppid = cpu_to_be32(1); /* * Indicate SIGUSR1 for crash initiated from kernel. @@ -130,7 +130,7 @@ static void fill_prstatus(struct elf_prstatus *prstatus, int pir, short sig; sig = kernel_initiated ? SIGUSR1 : SIGTERM; - prstatus->pr_cursig = cpu_to_be16(sig); + prstatus->common.pr_cursig = cpu_to_be16(sig); } } diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index 205b2e2648aa..0e36dfc9ccd6 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -365,7 +365,7 @@ static void *fill_cpu_elf_notes(void *ptr, int cpu, struct save_area *sa) memcpy(&nt_prstatus.pr_reg.gprs, sa->gprs, sizeof(sa->gprs)); memcpy(&nt_prstatus.pr_reg.psw, sa->psw, sizeof(sa->psw)); memcpy(&nt_prstatus.pr_reg.acrs, sa->acrs, sizeof(sa->acrs)); - nt_prstatus.pr_pid = cpu; + nt_prstatus.common.pr_pid = cpu; /* Prepare fpregset (floating point) note */ memset(&nt_fpregset, 0, sizeof(nt_fpregset)); memcpy(&nt_fpregset.fpc, &sa->fpc, sizeof(sa->fpc)); diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 8380478d3d92..4c1550b13899 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1495,7 +1495,7 @@ static void fill_note(struct memelfnote *note, const char *name, int type, * fill up all the fields in prstatus from the given task struct, except * registers which need to be filled up separately. */ -static void fill_prstatus(struct elf_prstatus *prstatus, +static void fill_prstatus(struct elf_prstatus_common *prstatus, struct task_struct *p, long signr) { prstatus->pr_info.si_signo = prstatus->pr_cursig = signr; @@ -1736,7 +1736,7 @@ static int fill_thread_core_info(struct elf_thread_core_info *t, * than being the whole note contents. We fill the reset in here. * We assume that regset 0 is NT_PRSTATUS. */ - fill_prstatus(&t->prstatus, t->task, signr); + fill_prstatus(&t->prstatus.common, t->task, signr); regset_get(t->task, &view->regsets[0], sizeof(t->prstatus.pr_reg), &t->prstatus.pr_reg); @@ -1958,7 +1958,7 @@ static int elf_dump_thread_status(long signr, struct elf_thread_status *t) struct task_struct *p = t->thread; t->num_notes = 0; - fill_prstatus(&t->prstatus, p, signr); + fill_prstatus(&t->prstatus.common, p, signr); elf_core_copy_task_regs(p, &t->prstatus.pr_reg); fill_note(&t->notes[0], "CORE", NT_PRSTATUS, sizeof(t->prstatus), @@ -2037,7 +2037,7 @@ static int fill_note_info(struct elfhdr *elf, int phdrs, } /* now collect the dump for the current */ memset(info->prstatus, 0, sizeof(*info->prstatus)); - fill_prstatus(info->prstatus, current, siginfo->si_signo); + fill_prstatus(&info->prstatus->common, current, siginfo->si_signo); elf_core_copy_regs(&info->prstatus->pr_reg, regs); /* Set up header */ diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index be4062b8ba75..03d81a14bcbf 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -1191,18 +1191,7 @@ static int elf_fdpic_map_file_by_direct_mmap(struct elf_fdpic_params *params, struct elf_prstatus_fdpic { - struct elf_siginfo pr_info; /* Info associated with signal */ - short pr_cursig; /* Current signal */ - unsigned long pr_sigpend; /* Set of pending signals */ - unsigned long pr_sighold; /* Set of held signals */ - pid_t pr_pid; - pid_t pr_ppid; - pid_t pr_pgrp; - pid_t pr_sid; - struct __kernel_old_timeval pr_utime; /* User time */ - struct __kernel_old_timeval pr_stime; /* System time */ - struct __kernel_old_timeval pr_cutime; /* Cumulative user time */ - struct __kernel_old_timeval pr_cstime; /* Cumulative system time */ + struct elf_prstatus_common common; elf_gregset_t pr_reg; /* GP registers */ /* When using FDPIC, the loadmap addresses need to be communicated * to GDB in order for GDB to do the necessary relocations. The @@ -1301,7 +1290,7 @@ static inline void fill_note(struct memelfnote *note, const char *name, int type * fill up all the fields in prstatus from the given task struct, except * registers which need to be filled up separately. */ -static void fill_prstatus(struct elf_prstatus_fdpic *prstatus, +static void fill_prstatus(struct elf_prstatus_common *prstatus, struct task_struct *p, long signr) { prstatus->pr_info.si_signo = prstatus->pr_cursig = signr; @@ -1332,9 +1321,6 @@ static void fill_prstatus(struct elf_prstatus_fdpic *prstatus, } prstatus->pr_cutime = ns_to_kernel_old_timeval(p->signal->cutime); prstatus->pr_cstime = ns_to_kernel_old_timeval(p->signal->cstime); - - prstatus->pr_exec_fdpic_loadmap = p->mm->context.exec_fdpic_loadmap; - prstatus->pr_interp_fdpic_loadmap = p->mm->context.interp_fdpic_loadmap; } static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p, @@ -1405,7 +1391,9 @@ static struct elf_thread_status *elf_dump_thread_status(long signr, struct task_ if (!t) return t; - fill_prstatus(&t->prstatus, p, signr); + fill_prstatus(&t->prstatus.common, p, signr); + t->prstatus.pr_exec_fdpic_loadmap = p->mm->context.exec_fdpic_loadmap; + t->prstatus.pr_interp_fdpic_loadmap = p->mm->context.interp_fdpic_loadmap; regset_get(p, &view->regsets[0], sizeof(t->prstatus.pr_reg), &t->prstatus.pr_reg); diff --git a/fs/compat_binfmt_elf.c b/fs/compat_binfmt_elf.c index 962df845ed51..feb48a5c2d44 100644 --- a/fs/compat_binfmt_elf.c +++ b/fs/compat_binfmt_elf.c @@ -50,6 +50,7 @@ * which requires asm/elf.h to define compat_elf_gregset_t et al. */ #define elf_prstatus compat_elf_prstatus +#define elf_prstatus_common compat_elf_prstatus_common #define elf_prpsinfo compat_elf_prpsinfo #undef ns_to_kernel_old_timeval diff --git a/include/linux/elfcore-compat.h b/include/linux/elfcore-compat.h index 10485f0c9740..4aeda5f1f038 100644 --- a/include/linux/elfcore-compat.h +++ b/include/linux/elfcore-compat.h @@ -17,7 +17,7 @@ struct compat_elf_siginfo compat_int_t si_errno; }; -struct compat_elf_prstatus +struct compat_elf_prstatus_common { struct compat_elf_siginfo pr_info; short pr_cursig; @@ -31,6 +31,11 @@ struct compat_elf_prstatus struct old_timeval32 pr_stime; struct old_timeval32 pr_cutime; struct old_timeval32 pr_cstime; +}; + +struct compat_elf_prstatus +{ + struct compat_elf_prstatus_common common; compat_elf_gregset_t pr_reg; compat_int_t pr_fpvalid; }; diff --git a/include/linux/elfcore.h b/include/linux/elfcore.h index de51c1bef27d..2aaa15779d50 100644 --- a/include/linux/elfcore.h +++ b/include/linux/elfcore.h @@ -29,7 +29,7 @@ struct elf_siginfo * the SVR4 structure, but more Linuxy, with things that Linux does * not support and which gdb doesn't really use excluded. */ -struct elf_prstatus +struct elf_prstatus_common { struct elf_siginfo pr_info; /* Info associated with signal */ short pr_cursig; /* Current signal */ @@ -43,6 +43,11 @@ struct elf_prstatus struct __kernel_old_timeval pr_stime; /* System time */ struct __kernel_old_timeval pr_cutime; /* Cumulative user time */ struct __kernel_old_timeval pr_cstime; /* Cumulative system time */ +}; + +struct elf_prstatus +{ + struct elf_prstatus_common common; elf_gregset_t pr_reg; /* GP registers */ int pr_fpvalid; /* True if math co-processor being used. */ }; diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 4f8efc278aa7..80905e5aa8ae 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1076,7 +1076,7 @@ void crash_save_cpu(struct pt_regs *regs, int cpu) if (!buf) return; memset(&prstatus, 0, sizeof(prstatus)); - prstatus.pr_pid = current->pid; + prstatus.common.pr_pid = current->pid; elf_core_copy_kernel_regs(&prstatus.pr_reg, regs); buf = append_elf_note(buf, KEXEC_CORE_NOTE_NAME, NT_PRSTATUS, &prstatus, sizeof(prstatus)); -- cgit v1.2.3 From 7facdc426f86c67e579e49e100943cbccc43e1c6 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 13 Jun 2020 23:03:25 -0400 Subject: [amd64] clean PRSTATUS_SIZE/SET_PR_FPVALID up properly To get rid of hardcoded size/offset in those macros we need to have a definition of i386 variant of struct elf_prstatus. However, we can't do that in asm/compat.h - the types needed for that are not there and adding an include of asm/user32.h into asm/compat.h would cause a lot of mess. That could be conveniently done in elfcore-compat.h, but currently there is nowhere to put arch-dependent parts of it - no asm/elfcore-compat.h. So we introduce a new file (asm/elfcore-compat.h, present on architectures that have CONFIG_ARCH_HAS_ELFCORE_COMPAT set, currently only on x86), have it pulled by linux/elfcore-compat.h and move the definitions there. As a side benefit, we don't need to worry about accidental inclusion of that file into binfmt_elf.c itself, so we don't need the dance with COMPAT_PRSTATUS_SIZE, etc. - only fs/compat_binfmt_elf.c will see that header. Signed-off-by: Al Viro --- arch/Kconfig | 3 +++ arch/x86/Kconfig | 1 + arch/x86/include/asm/compat.h | 14 -------------- arch/x86/include/asm/elfcore-compat.h | 31 +++++++++++++++++++++++++++++++ fs/compat_binfmt_elf.c | 8 -------- include/linux/elfcore-compat.h | 18 +++++++++++------- 6 files changed, 46 insertions(+), 29 deletions(-) create mode 100644 arch/x86/include/asm/elfcore-compat.h (limited to 'include/linux') diff --git a/arch/Kconfig b/arch/Kconfig index 78c6f05b10f9..a17ced73b23c 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1105,6 +1105,9 @@ config HAVE_ARCH_PFN_VALID config ARCH_SUPPORTS_DEBUG_PAGEALLOC bool +config ARCH_HAS_ELFCORE_COMPAT + bool + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig" diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 7b6dd10b162a..302a6b453c91 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -31,6 +31,7 @@ config X86_64 select MODULES_USE_ELF_RELA select NEED_DMA_MAP_STATE select SWIOTLB + select ARCH_HAS_ELFCORE_COMPAT config FORCE_DYNAMIC_FTRACE def_bool y diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h index 15cf0f831dee..be09c7eac89f 100644 --- a/arch/x86/include/asm/compat.h +++ b/arch/x86/include/asm/compat.h @@ -159,20 +159,6 @@ struct compat_shmid64_ds { compat_ulong_t __unused5; }; -/* - * The type of struct elf_prstatus.pr_reg in compatible core dumps. - */ -typedef struct user_regs_struct compat_elf_gregset_t; - -/* Full regset -- prstatus on x32, otherwise on ia32 */ -#define COMPAT_PRSTATUS_SIZE (user_64bit_mode(task_pt_regs(current)) \ - ? sizeof(struct compat_elf_prstatus) \ - : 144) -#define COMPAT_SET_PR_FPVALID(S) \ - (*(user_64bit_mode(task_pt_regs(current)) \ - ? &(S)->pr_fpvalid \ - : (int *)((void *)(S) + 140)) = 1) - #ifdef CONFIG_X86_X32_ABI #define COMPAT_USE_64BIT_TIME \ (!!(task_pt_regs(current)->orig_ax & __X32_SYSCALL_BIT)) diff --git a/arch/x86/include/asm/elfcore-compat.h b/arch/x86/include/asm/elfcore-compat.h new file mode 100644 index 000000000000..f1b6c7a8d8fc --- /dev/null +++ b/arch/x86/include/asm/elfcore-compat.h @@ -0,0 +1,31 @@ +#ifndef _ASM_X86_ELFCORE_COMPAT_H +#define _ASM_X86_ELFCORE_COMPAT_H + +#include + +/* + * On amd64 we have two 32bit ABIs - i386 and x32. The latter + * has bigger registers, so we use it for compat_elf_regset_t. + * The former uses i386_elf_prstatus and PRSTATUS_SIZE/SET_PR_FPVALID + * are used to choose the size and location of ->pr_fpvalid of + * the layout actually used. + */ +typedef struct user_regs_struct compat_elf_gregset_t; + +struct i386_elf_prstatus +{ + struct compat_elf_prstatus_common common; + struct user_regs_struct32 pr_reg; + compat_int_t pr_fpvalid; +}; + +#define PRSTATUS_SIZE \ + (user_64bit_mode(task_pt_regs(current)) \ + ? sizeof(struct compat_elf_prstatus) \ + : sizeof(struct i386_elf_prstatus)) +#define SET_PR_FPVALID(S) \ + (*(user_64bit_mode(task_pt_regs(current)) \ + ? &(S)->pr_fpvalid \ + : &((struct i386_elf_prstatus *)(S))->pr_fpvalid) = 1) + +#endif diff --git a/fs/compat_binfmt_elf.c b/fs/compat_binfmt_elf.c index feb48a5c2d44..a6321415aba0 100644 --- a/fs/compat_binfmt_elf.c +++ b/fs/compat_binfmt_elf.c @@ -96,14 +96,6 @@ #define ELF_EXEC_PAGESIZE COMPAT_ELF_EXEC_PAGESIZE #endif -#ifdef COMPAT_PRSTATUS_SIZE -#define PRSTATUS_SIZE COMPAT_PRSTATUS_SIZE -#endif - -#ifdef COMPAT_SET_PR_FPVALID -#define SET_PR_FPVALID(S) COMPAT_SET_PR_FPVALID(S) -#endif - #ifdef COMPAT_ELF_PLAT_INIT #undef ELF_PLAT_INIT #define ELF_PLAT_INIT COMPAT_ELF_PLAT_INIT diff --git a/include/linux/elfcore-compat.h b/include/linux/elfcore-compat.h index 4aeda5f1f038..e272c3d452ce 100644 --- a/include/linux/elfcore-compat.h +++ b/include/linux/elfcore-compat.h @@ -33,13 +33,6 @@ struct compat_elf_prstatus_common struct old_timeval32 pr_cstime; }; -struct compat_elf_prstatus -{ - struct compat_elf_prstatus_common common; - compat_elf_gregset_t pr_reg; - compat_int_t pr_fpvalid; -}; - struct compat_elf_prpsinfo { char pr_state; @@ -54,4 +47,15 @@ struct compat_elf_prpsinfo char pr_psargs[ELF_PRARGSZ]; }; +#ifdef CONFIG_ARCH_HAS_ELFCORE_COMPAT +#include +#endif + +struct compat_elf_prstatus +{ + struct compat_elf_prstatus_common common; + compat_elf_gregset_t pr_reg; + compat_int_t pr_fpvalid; +}; + #endif /* _LINUX_ELFCORE_COMPAT_H */ -- cgit v1.2.3 From ae5c2341ed3987bd434ed495bd4f3d8b2bc3e623 Mon Sep 17 00:00:00 2001 From: "Joel Fernandes (Google)" Date: Wed, 23 Sep 2020 11:22:09 -0400 Subject: rcu/segcblist: Add counters to segcblist datastructure Add counting of segment lengths of segmented callback list. This will be useful for a number of things such as knowing how big the ready-to-execute segment have gotten. The immediate benefit is ability to trace how the callbacks in the segmented callback list change. Also this patch remove hacks related to using donecbs's ->len field as a temporary variable to save the segmented callback list's length. This cannot be done anymore and is not needed. Also fix SRCU: The negative counting of the unsegmented list cannot be used to adjust the segmented one. To fix this, sample the unsegmented length in advance, and use it after CB execution to adjust the segmented list's length. Reviewed-by: Frederic Weisbecker Signed-off-by: Joel Fernandes (Google) Signed-off-by: Paul E. McKenney --- include/linux/rcu_segcblist.h | 1 + kernel/rcu/rcu_segcblist.c | 120 +++++++++++++++++++++++++++--------------- kernel/rcu/rcu_segcblist.h | 2 - kernel/rcu/srcutree.c | 5 +- 4 files changed, 82 insertions(+), 46 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index b36afe7b22c9..6c01f09a6456 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -72,6 +72,7 @@ struct rcu_segcblist { #else long len; #endif + long seglen[RCU_CBLIST_NSEGS]; u8 enabled; u8 offloaded; }; diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index 3cff8003f707..777780447887 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -7,10 +7,10 @@ * Authors: Paul E. McKenney */ -#include -#include +#include #include -#include +#include +#include #include "rcu_segcblist.h" @@ -88,6 +88,46 @@ static void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v) #endif } +/* Get the length of a segment of the rcu_segcblist structure. */ +static long rcu_segcblist_get_seglen(struct rcu_segcblist *rsclp, int seg) +{ + return READ_ONCE(rsclp->seglen[seg]); +} + +/* Set the length of a segment of the rcu_segcblist structure. */ +static void rcu_segcblist_set_seglen(struct rcu_segcblist *rsclp, int seg, long v) +{ + WRITE_ONCE(rsclp->seglen[seg], v); +} + +/* Increase the numeric length of a segment by a specified amount. */ +static void rcu_segcblist_add_seglen(struct rcu_segcblist *rsclp, int seg, long v) +{ + WRITE_ONCE(rsclp->seglen[seg], rsclp->seglen[seg] + v); +} + +/* Move from's segment length to to's segment. */ +static void rcu_segcblist_move_seglen(struct rcu_segcblist *rsclp, int from, int to) +{ + long len; + + if (from == to) + return; + + len = rcu_segcblist_get_seglen(rsclp, from); + if (!len) + return; + + rcu_segcblist_add_seglen(rsclp, to, len); + rcu_segcblist_set_seglen(rsclp, from, 0); +} + +/* Increment segment's length. */ +static void rcu_segcblist_inc_seglen(struct rcu_segcblist *rsclp, int seg) +{ + rcu_segcblist_add_seglen(rsclp, seg, 1); +} + /* * Increase the numeric length of an rcu_segcblist structure by the * specified amount, which can be negative. This can cause the ->len @@ -179,26 +219,6 @@ void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp) rcu_segcblist_add_len(rsclp, 1); } -/* - * Exchange the numeric length of the specified rcu_segcblist structure - * with the specified value. This can cause the ->len field to disagree - * with the actual number of callbacks on the structure. This exchange is - * fully ordered with respect to the callers accesses both before and after. - */ -static long rcu_segcblist_xchg_len(struct rcu_segcblist *rsclp, long v) -{ -#ifdef CONFIG_RCU_NOCB_CPU - return atomic_long_xchg(&rsclp->len, v); -#else - long ret = rsclp->len; - - smp_mb(); /* Up to the caller! */ - WRITE_ONCE(rsclp->len, v); - smp_mb(); /* Up to the caller! */ - return ret; -#endif -} - /* * Initialize an rcu_segcblist structure. */ @@ -209,8 +229,10 @@ void rcu_segcblist_init(struct rcu_segcblist *rsclp) BUILD_BUG_ON(RCU_NEXT_TAIL + 1 != ARRAY_SIZE(rsclp->gp_seq)); BUILD_BUG_ON(ARRAY_SIZE(rsclp->tails) != ARRAY_SIZE(rsclp->gp_seq)); rsclp->head = NULL; - for (i = 0; i < RCU_CBLIST_NSEGS; i++) + for (i = 0; i < RCU_CBLIST_NSEGS; i++) { rsclp->tails[i] = &rsclp->head; + rcu_segcblist_set_seglen(rsclp, i, 0); + } rcu_segcblist_set_len(rsclp, 0); rsclp->enabled = 1; } @@ -306,6 +328,7 @@ void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, { rcu_segcblist_inc_len(rsclp); smp_mb(); /* Ensure counts are updated before callback is enqueued. */ + rcu_segcblist_inc_seglen(rsclp, RCU_NEXT_TAIL); rhp->next = NULL; WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp); WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], &rhp->next); @@ -334,27 +357,13 @@ bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, for (i = RCU_NEXT_TAIL; i > RCU_DONE_TAIL; i--) if (rsclp->tails[i] != rsclp->tails[i - 1]) break; + rcu_segcblist_inc_seglen(rsclp, i); WRITE_ONCE(*rsclp->tails[i], rhp); for (; i <= RCU_NEXT_TAIL; i++) WRITE_ONCE(rsclp->tails[i], &rhp->next); return true; } -/* - * Extract only the counts from the specified rcu_segcblist structure, - * and place them in the specified rcu_cblist structure. This function - * supports both callback orphaning and invocation, hence the separation - * of counts and callbacks. (Callbacks ready for invocation must be - * orphaned and adopted separately from pending callbacks, but counts - * apply to all callbacks. Locking must be used to make sure that - * both orphaned-callbacks lists are consistent.) - */ -void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp, - struct rcu_cblist *rclp) -{ - rclp->len = rcu_segcblist_xchg_len(rsclp, 0); -} - /* * Extract only those callbacks ready to be invoked from the specified * rcu_segcblist structure and place them in the specified rcu_cblist @@ -367,6 +376,7 @@ void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, if (!rcu_segcblist_ready_cbs(rsclp)) return; /* Nothing to do. */ + rclp->len = rcu_segcblist_get_seglen(rsclp, RCU_DONE_TAIL); *rclp->tail = rsclp->head; WRITE_ONCE(rsclp->head, *rsclp->tails[RCU_DONE_TAIL]); WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL); @@ -374,6 +384,7 @@ void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, for (i = RCU_CBLIST_NSEGS - 1; i >= RCU_DONE_TAIL; i--) if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL]) WRITE_ONCE(rsclp->tails[i], &rsclp->head); + rcu_segcblist_set_seglen(rsclp, RCU_DONE_TAIL, 0); } /* @@ -390,11 +401,15 @@ void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, if (!rcu_segcblist_pend_cbs(rsclp)) return; /* Nothing to do. */ + rclp->len = 0; *rclp->tail = *rsclp->tails[RCU_DONE_TAIL]; rclp->tail = rsclp->tails[RCU_NEXT_TAIL]; WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL); - for (i = RCU_DONE_TAIL + 1; i < RCU_CBLIST_NSEGS; i++) + for (i = RCU_DONE_TAIL + 1; i < RCU_CBLIST_NSEGS; i++) { + rclp->len += rcu_segcblist_get_seglen(rsclp, i); WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]); + rcu_segcblist_set_seglen(rsclp, i, 0); + } } /* @@ -405,7 +420,6 @@ void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp) { rcu_segcblist_add_len(rsclp, rclp->len); - rclp->len = 0; } /* @@ -419,6 +433,7 @@ void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, if (!rclp->head) return; /* No callbacks to move. */ + rcu_segcblist_add_seglen(rsclp, RCU_DONE_TAIL, rclp->len); *rclp->tail = rsclp->head; WRITE_ONCE(rsclp->head, rclp->head); for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++) @@ -439,6 +454,8 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, { if (!rclp->head) return; /* Nothing to do. */ + + rcu_segcblist_add_seglen(rsclp, RCU_NEXT_TAIL, rclp->len); WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head); WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail); } @@ -463,6 +480,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) if (ULONG_CMP_LT(seq, rsclp->gp_seq[i])) break; WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]); + rcu_segcblist_move_seglen(rsclp, i, RCU_DONE_TAIL); } /* If no callbacks moved, nothing more need be done. */ @@ -483,6 +501,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL]) break; /* No more callbacks. */ WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]); + rcu_segcblist_move_seglen(rsclp, i, j); rsclp->gp_seq[j] = rsclp->gp_seq[i]; } } @@ -504,7 +523,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) */ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) { - int i; + int i, j; WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) @@ -547,6 +566,10 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) if (rcu_segcblist_restempty(rsclp, i) || ++i >= RCU_NEXT_TAIL) return false; + /* Accounting: everything below i is about to get merged into i. */ + for (j = i + 1; j <= RCU_NEXT_TAIL; j++) + rcu_segcblist_move_seglen(rsclp, j, i); + /* * Merge all later callbacks, including newly arrived callbacks, * into the segment located by the for-loop above. Assign "seq" @@ -574,13 +597,24 @@ void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, struct rcu_cblist donecbs; struct rcu_cblist pendcbs; + lockdep_assert_cpus_held(); + rcu_cblist_init(&donecbs); rcu_cblist_init(&pendcbs); - rcu_segcblist_extract_count(src_rsclp, &donecbs); + rcu_segcblist_extract_done_cbs(src_rsclp, &donecbs); rcu_segcblist_extract_pend_cbs(src_rsclp, &pendcbs); + + /* + * No need smp_mb() before setting length to 0, because CPU hotplug + * lock excludes rcu_barrier. + */ + rcu_segcblist_set_len(src_rsclp, 0); + rcu_segcblist_insert_count(dst_rsclp, &donecbs); + rcu_segcblist_insert_count(dst_rsclp, &pendcbs); rcu_segcblist_insert_done_cbs(dst_rsclp, &donecbs); rcu_segcblist_insert_pend_cbs(dst_rsclp, &pendcbs); + rcu_segcblist_init(src_rsclp); } diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index 1d2d61406463..cd35c9faaf51 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h @@ -89,8 +89,6 @@ void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, struct rcu_head *rhp); bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, struct rcu_head *rhp); -void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp, - struct rcu_cblist *rclp); void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp); void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp, diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 0f23d20d485a..79b7081143a7 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -1160,6 +1160,7 @@ static void srcu_advance_state(struct srcu_struct *ssp) */ static void srcu_invoke_callbacks(struct work_struct *work) { + long len; bool more; struct rcu_cblist ready_cbs; struct rcu_head *rhp; @@ -1182,6 +1183,7 @@ static void srcu_invoke_callbacks(struct work_struct *work) /* We are on the job! Extract and invoke ready callbacks. */ sdp->srcu_cblist_invoking = true; rcu_segcblist_extract_done_cbs(&sdp->srcu_cblist, &ready_cbs); + len = ready_cbs.len; spin_unlock_irq_rcu_node(sdp); rhp = rcu_cblist_dequeue(&ready_cbs); for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) { @@ -1190,13 +1192,14 @@ static void srcu_invoke_callbacks(struct work_struct *work) rhp->func(rhp); local_bh_enable(); } + WARN_ON_ONCE(ready_cbs.len); /* * Update counts, accelerate new callbacks, and if needed, * schedule another round of callback invocation. */ spin_lock_irq_rcu_node(sdp); - rcu_segcblist_insert_count(&sdp->srcu_cblist, &ready_cbs); + rcu_segcblist_add_len(&sdp->srcu_cblist, -len); (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, rcu_seq_snap(&ssp->srcu_gp_seq)); sdp->srcu_cblist_invoking = false; -- cgit v1.2.3 From 65e560327fe68153a9ad7452d5fd3171a1927d33 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 13 Nov 2020 13:13:16 +0100 Subject: rcu/nocb: Turn enabled/offload states into a common flag This commit gathers the rcu_segcblist ->enabled and ->offloaded property field into a single ->flags bitmask to avoid further proliferation of individual u8 fields in the structure. This change prepares for the state formerly known as ->offloaded state to be modified at runtime. Cc: Josh Triplett Cc: Steven Rostedt Cc: Mathieu Desnoyers Cc: Lai Jiangshan Cc: Joel Fernandes Cc: Neeraj Upadhyay Cc: Thomas Gleixner Inspired-by: Paul E. McKenney Tested-by: Boqun Feng Signed-off-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcu_segcblist.h | 6 ++++-- kernel/rcu/rcu_segcblist.c | 6 +++--- kernel/rcu/rcu_segcblist.h | 23 +++++++++++++++++++++-- 3 files changed, 28 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index 6c01f09a6456..4714b0263c76 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -63,6 +63,9 @@ struct rcu_cblist { #define RCU_NEXT_TAIL 3 #define RCU_CBLIST_NSEGS 4 +#define SEGCBLIST_ENABLED BIT(0) +#define SEGCBLIST_OFFLOADED BIT(1) + struct rcu_segcblist { struct rcu_head *head; struct rcu_head **tails[RCU_CBLIST_NSEGS]; @@ -73,8 +76,7 @@ struct rcu_segcblist { long len; #endif long seglen[RCU_CBLIST_NSEGS]; - u8 enabled; - u8 offloaded; + u8 flags; }; #define RCU_SEGCBLIST_INITIALIZER(n) \ diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index 89e0dff890bf..934945d72ca5 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -246,7 +246,7 @@ void rcu_segcblist_init(struct rcu_segcblist *rsclp) rcu_segcblist_set_seglen(rsclp, i, 0); } rcu_segcblist_set_len(rsclp, 0); - rsclp->enabled = 1; + rcu_segcblist_set_flags(rsclp, SEGCBLIST_ENABLED); } /* @@ -257,7 +257,7 @@ void rcu_segcblist_disable(struct rcu_segcblist *rsclp) { WARN_ON_ONCE(!rcu_segcblist_empty(rsclp)); WARN_ON_ONCE(rcu_segcblist_n_cbs(rsclp)); - rsclp->enabled = 0; + rcu_segcblist_clear_flags(rsclp, SEGCBLIST_ENABLED); } /* @@ -266,7 +266,7 @@ void rcu_segcblist_disable(struct rcu_segcblist *rsclp) */ void rcu_segcblist_offload(struct rcu_segcblist *rsclp) { - rsclp->offloaded = 1; + rcu_segcblist_set_flags(rsclp, SEGCBLIST_OFFLOADED); } /* diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index 18e101de8747..ff372db09f8b 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h @@ -53,19 +53,38 @@ static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp) #endif } +static inline void rcu_segcblist_set_flags(struct rcu_segcblist *rsclp, + int flags) +{ + rsclp->flags |= flags; +} + +static inline void rcu_segcblist_clear_flags(struct rcu_segcblist *rsclp, + int flags) +{ + rsclp->flags &= ~flags; +} + +static inline bool rcu_segcblist_test_flags(struct rcu_segcblist *rsclp, + int flags) +{ + return READ_ONCE(rsclp->flags) & flags; +} + /* * Is the specified rcu_segcblist enabled, for example, not corresponding * to an offline CPU? */ static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) { - return rsclp->enabled; + return rcu_segcblist_test_flags(rsclp, SEGCBLIST_ENABLED); } /* Is the specified rcu_segcblist offloaded? */ static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) { - return IS_ENABLED(CONFIG_RCU_NOCB_CPU) && rsclp->offloaded; + return IS_ENABLED(CONFIG_RCU_NOCB_CPU) && + rcu_segcblist_test_flags(rsclp, SEGCBLIST_OFFLOADED); } /* -- cgit v1.2.3 From 8d346d438f93b5344e99d429727ec9c2f392d4ec Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 13 Nov 2020 13:13:17 +0100 Subject: rcu/nocb: Provide basic callback offloading state machine bits Offloading and de-offloading RCU callback processes must be done carefully. There must never be a time at which callback processing is disabled because the task driving the offloading or de-offloading might be preempted or otherwise stalled at that point in time, which would result in OOM due to calbacks piling up indefinitely. This implies that there will be times during which a given CPU's callbacks might be concurrently invoked by both that CPU's RCU_SOFTIRQ handler (or, equivalently, that CPU's rcuc kthread) and by that CPU's rcuo kthread. This situation could fatally confuse both rcu_barrier() and the CPU-hotplug offlining process, so these must be excluded during any concurrent-callback-invocation period. In addition, during times of concurrent callback invocation, changes to ->cblist must be protected both as needed for RCU_SOFTIRQ and as needed for the rcuo kthread. This commit therefore defines and documents the states for a state machine that coordinates offloading and deoffloading. Cc: Josh Triplett Cc: Steven Rostedt Cc: Mathieu Desnoyers Cc: Lai Jiangshan Cc: Joel Fernandes Cc: Neeraj Upadhyay Cc: Thomas Gleixner Inspired-by: Paul E. McKenney Tested-by: Boqun Feng Signed-off-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcu_segcblist.h | 115 +++++++++++++++++++++++++++++++++++++++++- kernel/rcu/rcu_segcblist.c | 1 + kernel/rcu/rcu_segcblist.h | 12 ++++- kernel/rcu/tree.c | 3 ++ 4 files changed, 128 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index 4714b0263c76..8afe886e85f1 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -63,8 +63,121 @@ struct rcu_cblist { #define RCU_NEXT_TAIL 3 #define RCU_CBLIST_NSEGS 4 + +/* + * ==NOCB Offloading state machine== + * + * + * ---------------------------------------------------------------------------- + * | SEGCBLIST_SOFTIRQ_ONLY | + * | | + * | Callbacks processed by rcu_core() from softirqs or local | + * | rcuc kthread, without holding nocb_lock. | + * ---------------------------------------------------------------------------- + * | + * v + * ---------------------------------------------------------------------------- + * | SEGCBLIST_OFFLOADED | + * | | + * | Callbacks processed by rcu_core() from softirqs or local | + * | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads, | + * | allowing nocb_timer to be armed. | + * ---------------------------------------------------------------------------- + * | + * v + * ----------------------------------- + * | | + * v v + * --------------------------------------- ----------------------------------| + * | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | | + * | SEGCBLIST_KTHREAD_CB | | SEGCBLIST_KTHREAD_GP | + * | | | | + * | | | | + * | CB kthread woke up and | | GP kthread woke up and | + * | acknowledged SEGCBLIST_OFFLOADED. | | acknowledged SEGCBLIST_OFFLOADED| + * | Processes callbacks concurrently | | | + * | with rcu_core(), holding | | | + * | nocb_lock. | | | + * --------------------------------------- ----------------------------------- + * | | + * ----------------------------------- + * | + * v + * |--------------------------------------------------------------------------| + * | SEGCBLIST_OFFLOADED | | + * | SEGCBLIST_KTHREAD_CB | | + * | SEGCBLIST_KTHREAD_GP | + * | | + * | Kthreads handle callbacks holding nocb_lock, local rcu_core() stops | + * | handling callbacks. | + * ---------------------------------------------------------------------------- + */ + + + +/* + * ==NOCB De-Offloading state machine== + * + * + * |--------------------------------------------------------------------------| + * | SEGCBLIST_OFFLOADED | | + * | SEGCBLIST_KTHREAD_CB | | + * | SEGCBLIST_KTHREAD_GP | + * | | + * | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() | + * | ignores callbacks. | + * ---------------------------------------------------------------------------- + * | + * v + * |--------------------------------------------------------------------------| + * | SEGCBLIST_KTHREAD_CB | | + * | SEGCBLIST_KTHREAD_GP | + * | | + * | CB/GP kthreads and local rcu_core() handle callbacks concurrently | + * | holding nocb_lock. Wake up CB and GP kthreads if necessary. | + * ---------------------------------------------------------------------------- + * | + * v + * ----------------------------------- + * | | + * v v + * ---------------------------------------------------------------------------| + * | | + * | SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP | + * | | | + * | GP kthread woke up and | CB kthread woke up and | + * | acknowledged the fact that | acknowledged the fact that | + * | SEGCBLIST_OFFLOADED got cleared. | SEGCBLIST_OFFLOADED got cleared. | + * | | The CB kthread goes to sleep | + * | The callbacks from the target CPU | until it ever gets re-offloaded. | + * | will be ignored from the GP kthread | | + * | loop. | | + * ---------------------------------------------------------------------------- + * | | + * ----------------------------------- + * | + * v + * ---------------------------------------------------------------------------- + * | 0 | + * | | + * | Callbacks processed by rcu_core() from softirqs or local | + * | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed. | + * | Flush pending nocb_timer. Flush nocb bypass callbacks. | + * ---------------------------------------------------------------------------- + * | + * v + * ---------------------------------------------------------------------------- + * | SEGCBLIST_SOFTIRQ_ONLY | + * | | + * | Callbacks processed by rcu_core() from softirqs or local | + * | rcuc kthread, without holding nocb_lock. | + * ---------------------------------------------------------------------------- + */ #define SEGCBLIST_ENABLED BIT(0) -#define SEGCBLIST_OFFLOADED BIT(1) +#define SEGCBLIST_SOFTIRQ_ONLY BIT(1) +#define SEGCBLIST_KTHREAD_CB BIT(2) +#define SEGCBLIST_KTHREAD_GP BIT(3) +#define SEGCBLIST_OFFLOADED BIT(4) struct rcu_segcblist { struct rcu_head *head; diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index 934945d72ca5..7fc6362625b2 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -266,6 +266,7 @@ void rcu_segcblist_disable(struct rcu_segcblist *rsclp) */ void rcu_segcblist_offload(struct rcu_segcblist *rsclp) { + rcu_segcblist_clear_flags(rsclp, SEGCBLIST_SOFTIRQ_ONLY); rcu_segcblist_set_flags(rsclp, SEGCBLIST_OFFLOADED); } diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index ff372db09f8b..e05952ab9b87 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h @@ -83,8 +83,16 @@ static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp) /* Is the specified rcu_segcblist offloaded? */ static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp) { - return IS_ENABLED(CONFIG_RCU_NOCB_CPU) && - rcu_segcblist_test_flags(rsclp, SEGCBLIST_OFFLOADED); + if (IS_ENABLED(CONFIG_RCU_NOCB_CPU)) { + /* + * Complete de-offloading happens only when SEGCBLIST_SOFTIRQ_ONLY + * is set. + */ + if (!rcu_segcblist_test_flags(rsclp, SEGCBLIST_SOFTIRQ_ONLY)) + return true; + } + + return false; } /* diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 8086c0467c15..7cfc2e8a2500 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -83,6 +83,9 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { .dynticks_nesting = 1, .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE, .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR), +#ifdef CONFIG_RCU_NOCB_CPU + .cblist.flags = SEGCBLIST_SOFTIRQ_ONLY, +#endif }; static struct rcu_state rcu_state = { .level = { &rcu_state.node[0] }, -- cgit v1.2.3 From d97b078182406c0bd0aacd36fc0a693e118e608f Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 13 Nov 2020 13:13:19 +0100 Subject: rcu/nocb: De-offloading CB kthread To de-offload callback processing back onto a CPU, it is necessary to clear SEGCBLIST_OFFLOAD and notify the nocb CB kthread, which will then clear its own bit flag and go to sleep to stop handling callbacks. This commit makes that change. It will also be necessary to notify the nocb GP kthread in this same way, which is the subject of a follow-on commit. Cc: Josh Triplett Cc: Steven Rostedt Cc: Mathieu Desnoyers Cc: Lai Jiangshan Cc: Joel Fernandes Cc: Neeraj Upadhyay Cc: Thomas Gleixner Inspired-by: Paul E. McKenney Tested-by: Boqun Feng Signed-off-by: Frederic Weisbecker [ paulmck: Add export per kernel test robot feedback. ] Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 2 + kernel/rcu/rcu_segcblist.c | 10 ++-- kernel/rcu/rcu_segcblist.h | 2 +- kernel/rcu/tree.h | 1 + kernel/rcu/tree_plugin.h | 130 ++++++++++++++++++++++++++++++++++++++------- 5 files changed, 123 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index de0826411311..40266eb418b6 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -104,8 +104,10 @@ static inline void rcu_user_exit(void) { } #ifdef CONFIG_RCU_NOCB_CPU void rcu_init_nohz(void); +int rcu_nocb_cpu_deoffload(int cpu); #else /* #ifdef CONFIG_RCU_NOCB_CPU */ static inline void rcu_init_nohz(void) { } +static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; } #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ /** diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index 7fc6362625b2..7f181c9675f7 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -264,10 +264,14 @@ void rcu_segcblist_disable(struct rcu_segcblist *rsclp) * Mark the specified rcu_segcblist structure as offloaded. This * structure must be empty. */ -void rcu_segcblist_offload(struct rcu_segcblist *rsclp) +void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload) { - rcu_segcblist_clear_flags(rsclp, SEGCBLIST_SOFTIRQ_ONLY); - rcu_segcblist_set_flags(rsclp, SEGCBLIST_OFFLOADED); + if (offload) { + rcu_segcblist_clear_flags(rsclp, SEGCBLIST_SOFTIRQ_ONLY); + rcu_segcblist_set_flags(rsclp, SEGCBLIST_OFFLOADED); + } else { + rcu_segcblist_clear_flags(rsclp, SEGCBLIST_OFFLOADED); + } } /* diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index e05952ab9b87..28c9a5225afc 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h @@ -109,7 +109,7 @@ void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp); void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v); void rcu_segcblist_init(struct rcu_segcblist *rsclp); void rcu_segcblist_disable(struct rcu_segcblist *rsclp); -void rcu_segcblist_offload(struct rcu_segcblist *rsclp); +void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload); bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 7708ed161f4a..e0deb4829847 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -201,6 +201,7 @@ struct rcu_data { /* 5) Callback offloading. */ #ifdef CONFIG_RCU_NOCB_CPU struct swait_queue_head nocb_cb_wq; /* For nocb kthreads to sleep on. */ + struct swait_queue_head nocb_state_wq; /* For offloading state changes */ struct task_struct *nocb_gp_kthread; raw_spinlock_t nocb_lock; /* Guard following pair of fields. */ atomic_t nocb_lock_contended; /* Contention experienced. */ diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 7e291ce0a1d6..1b870d0d2445 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -2081,16 +2081,29 @@ static int rcu_nocb_gp_kthread(void *arg) return 0; } +static inline bool nocb_cb_can_run(struct rcu_data *rdp) +{ + u8 flags = SEGCBLIST_OFFLOADED | SEGCBLIST_KTHREAD_CB; + return rcu_segcblist_test_flags(&rdp->cblist, flags); +} + +static inline bool nocb_cb_wait_cond(struct rcu_data *rdp) +{ + return nocb_cb_can_run(rdp) && !READ_ONCE(rdp->nocb_cb_sleep); +} + /* * Invoke any ready callbacks from the corresponding no-CBs CPU, * then, if there are no more, wait for more to appear. */ static void nocb_cb_wait(struct rcu_data *rdp) { + struct rcu_segcblist *cblist = &rdp->cblist; + struct rcu_node *rnp = rdp->mynode; + bool needwake_state = false; + bool needwake_gp = false; unsigned long cur_gp_seq; unsigned long flags; - bool needwake_gp = false; - struct rcu_node *rnp = rdp->mynode; local_irq_save(flags); rcu_momentary_dyntick_idle(); @@ -2100,32 +2113,50 @@ static void nocb_cb_wait(struct rcu_data *rdp) local_bh_enable(); lockdep_assert_irqs_enabled(); rcu_nocb_lock_irqsave(rdp, flags); - if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && + if (rcu_segcblist_nextgp(cblist, &cur_gp_seq) && rcu_seq_done(&rnp->gp_seq, cur_gp_seq) && raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */ needwake_gp = rcu_advance_cbs(rdp->mynode, rdp); raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ } - if (rcu_segcblist_ready_cbs(&rdp->cblist)) { - rcu_nocb_unlock_irqrestore(rdp, flags); - if (needwake_gp) - rcu_gp_kthread_wake(); - return; - } - trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep")); WRITE_ONCE(rdp->nocb_cb_sleep, true); + + if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED)) { + if (rcu_segcblist_ready_cbs(cblist)) + WRITE_ONCE(rdp->nocb_cb_sleep, false); + } else { + /* + * De-offloading. Clear our flag and notify the de-offload worker. + * We won't touch the callbacks and keep sleeping until we ever + * get re-offloaded. + */ + WARN_ON_ONCE(!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)); + rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_CB); + if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) + needwake_state = true; + } + + if (rdp->nocb_cb_sleep) + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep")); + rcu_nocb_unlock_irqrestore(rdp, flags); if (needwake_gp) rcu_gp_kthread_wake(); - swait_event_interruptible_exclusive(rdp->nocb_cb_wq, - !READ_ONCE(rdp->nocb_cb_sleep)); - if (!smp_load_acquire(&rdp->nocb_cb_sleep)) { /* VVV */ + + if (needwake_state) + swake_up_one(&rdp->nocb_state_wq); + + do { + swait_event_interruptible_exclusive(rdp->nocb_cb_wq, + nocb_cb_wait_cond(rdp)); + /* ^^^ Ensure CB invocation follows _sleep test. */ - return; - } - WARN_ON(signal_pending(current)); - trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); + if (smp_load_acquire(&rdp->nocb_cb_sleep)) { + WARN_ON(signal_pending(current)); + trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); + } + } while (!nocb_cb_can_run(rdp)); } /* @@ -2187,6 +2218,67 @@ static void do_nocb_deferred_wakeup(struct rcu_data *rdp) do_nocb_deferred_wakeup_common(rdp); } +static int __rcu_nocb_rdp_deoffload(struct rcu_data *rdp) +{ + struct rcu_segcblist *cblist = &rdp->cblist; + bool wake_cb = false; + unsigned long flags; + + printk("De-offloading %d\n", rdp->cpu); + + rcu_nocb_lock_irqsave(rdp, flags); + rcu_segcblist_offload(cblist, false); + + if (rdp->nocb_cb_sleep) { + rdp->nocb_cb_sleep = false; + wake_cb = true; + } + rcu_nocb_unlock_irqrestore(rdp, flags); + + if (wake_cb) + swake_up_one(&rdp->nocb_cb_wq); + + swait_event_exclusive(rdp->nocb_state_wq, + !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)); + + return 0; +} + +static long rcu_nocb_rdp_deoffload(void *arg) +{ + struct rcu_data *rdp = arg; + + WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id()); + return __rcu_nocb_rdp_deoffload(rdp); +} + +int rcu_nocb_cpu_deoffload(int cpu) +{ + struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); + int ret = 0; + + if (rdp == rdp->nocb_gp_rdp) { + pr_info("Can't deoffload an rdp GP leader (yet)\n"); + return -EINVAL; + } + mutex_lock(&rcu_state.barrier_mutex); + cpus_read_lock(); + if (rcu_segcblist_is_offloaded(&rdp->cblist)) { + if (cpu_online(cpu)) { + ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp); + } else { + ret = __rcu_nocb_rdp_deoffload(rdp); + } + if (!ret) + cpumask_clear_cpu(cpu, rcu_nocb_mask); + } + cpus_read_unlock(); + mutex_unlock(&rcu_state.barrier_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(rcu_nocb_cpu_deoffload); + void __init rcu_init_nohz(void) { int cpu; @@ -2229,7 +2321,8 @@ void __init rcu_init_nohz(void) rdp = per_cpu_ptr(&rcu_data, cpu); if (rcu_segcblist_empty(&rdp->cblist)) rcu_segcblist_init(&rdp->cblist); - rcu_segcblist_offload(&rdp->cblist); + rcu_segcblist_offload(&rdp->cblist, true); + rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB); } rcu_organize_nocb_kthreads(); } @@ -2239,6 +2332,7 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) { init_swait_queue_head(&rdp->nocb_cb_wq); init_swait_queue_head(&rdp->nocb_gp_wq); + init_swait_queue_head(&rdp->nocb_state_wq); raw_spin_lock_init(&rdp->nocb_lock); raw_spin_lock_init(&rdp->nocb_bypass_lock); raw_spin_lock_init(&rdp->nocb_gp_lock); -- cgit v1.2.3 From 254e11efde66ca0a0ce0c99a62c377314b5984ff Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 13 Nov 2020 13:13:22 +0100 Subject: rcu/nocb: Re-offload support To re-offload the callback processing off of a CPU, it is necessary to clear SEGCBLIST_SOFTIRQ_ONLY, set SEGCBLIST_OFFLOADED, and then notify both the CB and GP kthreads so that they both set their own bit flag and start processing the callbacks remotely. The re-offloading worker is then notified that it can stop the RCU_SOFTIRQ handler (or rcuc kthread, as the case may be) from processing the callbacks locally. Ordering must be carefully enforced so that the callbacks that used to be processed locally without locking will have the same ordering properties when they are invoked by the nocb CB and GP kthreads. This commit makes this change. Cc: Josh Triplett Cc: Steven Rostedt Cc: Mathieu Desnoyers Cc: Lai Jiangshan Cc: Joel Fernandes Cc: Neeraj Upadhyay Cc: Thomas Gleixner Inspired-by: Paul E. McKenney Tested-by: Boqun Feng Signed-off-by: Frederic Weisbecker [ paulmck: Export rcu_nocb_cpu_offload(). ] Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 2 + kernel/rcu/tree_plugin.h | 158 ++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 138 insertions(+), 22 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 40266eb418b6..e0ee52e2756d 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -104,9 +104,11 @@ static inline void rcu_user_exit(void) { } #ifdef CONFIG_RCU_NOCB_CPU void rcu_init_nohz(void); +int rcu_nocb_cpu_offload(int cpu); int rcu_nocb_cpu_deoffload(int cpu); #else /* #ifdef CONFIG_RCU_NOCB_CPU */ static inline void rcu_init_nohz(void) { } +static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; } static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; } #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index fe46e7008667..03ae1ce4790f 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1928,6 +1928,20 @@ static void do_nocb_bypass_wakeup_timer(struct timer_list *t) __call_rcu_nocb_wake(rdp, true, flags); } +/* + * Check if we ignore this rdp. + * + * We check that without holding the nocb lock but + * we make sure not to miss a freshly offloaded rdp + * with the current ordering: + * + * rdp_offload_toggle() nocb_gp_enabled_cb() + * ------------------------- ---------------------------- + * WRITE flags LOCK nocb_gp_lock + * LOCK nocb_gp_lock READ/WRITE nocb_gp_sleep + * READ/WRITE nocb_gp_sleep UNLOCK nocb_gp_lock + * UNLOCK nocb_gp_lock READ flags + */ static inline bool nocb_gp_enabled_cb(struct rcu_data *rdp) { u8 flags = SEGCBLIST_OFFLOADED | SEGCBLIST_KTHREAD_GP; @@ -1940,6 +1954,11 @@ static inline bool nocb_gp_update_state(struct rcu_data *rdp, bool *needwake_sta struct rcu_segcblist *cblist = &rdp->cblist; if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED)) { + if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) { + rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_GP); + if (rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) + *needwake_state = true; + } return true; } else { /* @@ -2003,6 +2022,8 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { rcu_nocb_unlock_irqrestore(rdp, flags); + if (needwake_state) + swake_up_one(&rdp->nocb_state_wq); continue; /* No callbacks here, try next. */ } if (bypass_ncbs) { @@ -2054,6 +2075,8 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) } if (needwake_gp) rcu_gp_kthread_wake(); + if (needwake_state) + swake_up_one(&rdp->nocb_state_wq); } my_rdp->nocb_gp_bypass = bypass; @@ -2159,6 +2182,11 @@ static void nocb_cb_wait(struct rcu_data *rdp) WRITE_ONCE(rdp->nocb_cb_sleep, true); if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED)) { + if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) { + rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_CB); + if (rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) + needwake_state = true; + } if (rcu_segcblist_ready_cbs(cblist)) WRITE_ONCE(rdp->nocb_cb_sleep, false); } else { @@ -2254,35 +2282,25 @@ static void do_nocb_deferred_wakeup(struct rcu_data *rdp) do_nocb_deferred_wakeup_common(rdp); } -static int __rcu_nocb_rdp_deoffload(struct rcu_data *rdp) +static int rdp_offload_toggle(struct rcu_data *rdp, + bool offload, unsigned long flags) + __releases(rdp->nocb_lock) { struct rcu_segcblist *cblist = &rdp->cblist; struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; - bool wake_cb = false, wake_gp = false; - unsigned long flags; - - printk("De-offloading %d\n", rdp->cpu); - - rcu_nocb_lock_irqsave(rdp, flags); - /* - * If there are still pending work offloaded, the offline - * CPU won't help much handling them. - */ - if (cpu_is_offline(rdp->cpu) && !rcu_segcblist_empty(&rdp->cblist)) { - rcu_nocb_unlock_irqrestore(rdp, flags); - return -EBUSY; - } + bool wake_gp = false; - rcu_segcblist_offload(cblist, false); + rcu_segcblist_offload(cblist, offload); - if (rdp->nocb_cb_sleep) { + if (rdp->nocb_cb_sleep) rdp->nocb_cb_sleep = false; - wake_cb = true; - } rcu_nocb_unlock_irqrestore(rdp, flags); - if (wake_cb) - swake_up_one(&rdp->nocb_cb_wq); + /* + * Ignore former value of nocb_cb_sleep and force wake up as it could + * have been spuriously set to false already. + */ + swake_up_one(&rdp->nocb_cb_wq); raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags); if (rdp_gp->nocb_gp_sleep) { @@ -2294,10 +2312,32 @@ static int __rcu_nocb_rdp_deoffload(struct rcu_data *rdp) if (wake_gp) wake_up_process(rdp_gp->nocb_gp_kthread); + return 0; +} + +static int __rcu_nocb_rdp_deoffload(struct rcu_data *rdp) +{ + struct rcu_segcblist *cblist = &rdp->cblist; + unsigned long flags; + int ret; + + printk("De-offloading %d\n", rdp->cpu); + + rcu_nocb_lock_irqsave(rdp, flags); + /* + * If there are still pending work offloaded, the offline + * CPU won't help much handling them. + */ + if (cpu_is_offline(rdp->cpu) && !rcu_segcblist_empty(&rdp->cblist)) { + rcu_nocb_unlock_irqrestore(rdp, flags); + return -EBUSY; + } + + ret = rdp_offload_toggle(rdp, false, flags); swait_event_exclusive(rdp->nocb_state_wq, !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP)); - return 0; + return ret; } static long rcu_nocb_rdp_deoffload(void *arg) @@ -2335,6 +2375,80 @@ int rcu_nocb_cpu_deoffload(int cpu) } EXPORT_SYMBOL_GPL(rcu_nocb_cpu_deoffload); +static int __rcu_nocb_rdp_offload(struct rcu_data *rdp) +{ + struct rcu_segcblist *cblist = &rdp->cblist; + unsigned long flags; + int ret; + + /* + * For now we only support re-offload, ie: the rdp must have been + * offloaded on boot first. + */ + if (!rdp->nocb_gp_rdp) + return -EINVAL; + + printk("Offloading %d\n", rdp->cpu); + /* + * Can't use rcu_nocb_lock_irqsave() while we are in + * SEGCBLIST_SOFTIRQ_ONLY mode. + */ + raw_spin_lock_irqsave(&rdp->nocb_lock, flags); + /* + * We didn't take the nocb lock while working on the + * rdp->cblist in SEGCBLIST_SOFTIRQ_ONLY mode. + * Every modifications that have been done previously on + * rdp->cblist must be visible remotely by the nocb kthreads + * upon wake up after reading the cblist flags. + * + * The layout against nocb_lock enforces that ordering: + * + * __rcu_nocb_rdp_offload() nocb_cb_wait()/nocb_gp_wait() + * ------------------------- ---------------------------- + * WRITE callbacks rcu_nocb_lock() + * rcu_nocb_lock() READ flags + * WRITE flags READ callbacks + * rcu_nocb_unlock() rcu_nocb_unlock() + */ + ret = rdp_offload_toggle(rdp, true, flags); + swait_event_exclusive(rdp->nocb_state_wq, + rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB) && + rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)); + + return ret; +} + +static long rcu_nocb_rdp_offload(void *arg) +{ + struct rcu_data *rdp = arg; + + WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id()); + return __rcu_nocb_rdp_offload(rdp); +} + +int rcu_nocb_cpu_offload(int cpu) +{ + struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); + int ret = 0; + + mutex_lock(&rcu_state.barrier_mutex); + cpus_read_lock(); + if (!rcu_segcblist_is_offloaded(&rdp->cblist)) { + if (cpu_online(cpu)) { + ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp); + } else { + ret = __rcu_nocb_rdp_offload(rdp); + } + if (!ret) + cpumask_set_cpu(cpu, rcu_nocb_mask); + } + cpus_read_unlock(); + mutex_unlock(&rcu_state.barrier_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(rcu_nocb_cpu_offload); + void __init rcu_init_nohz(void) { int cpu; -- cgit v1.2.3 From 43759fe5a137389e94ed6d4680c3c63c17273158 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 11 Nov 2020 23:53:13 +0100 Subject: cpu/hotplug: Add lockdep_is_cpus_held() This commit adds a lockdep_is_cpus_held() function to verify that the proper locks are held and that various operations are running in the correct context. Signed-off-by: Frederic Weisbecker Cc: Paul E. McKenney Cc: Josh Triplett Cc: Steven Rostedt Cc: Mathieu Desnoyers Cc: Lai Jiangshan Cc: Joel Fernandes Cc: Neeraj Upadhyay Cc: Thomas Gleixner Cc: Boqun Feng Signed-off-by: Paul E. McKenney --- include/linux/cpu.h | 2 ++ kernel/cpu.c | 7 +++++++ 2 files changed, 9 insertions(+) (limited to 'include/linux') diff --git a/include/linux/cpu.h b/include/linux/cpu.h index d6428aaf67e7..3aaa0687e8df 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -111,6 +111,8 @@ static inline void cpu_maps_update_done(void) #endif /* CONFIG_SMP */ extern struct bus_type cpu_subsys; +extern int lockdep_is_cpus_held(void); + #ifdef CONFIG_HOTPLUG_CPU extern void cpus_write_lock(void); extern void cpus_write_unlock(void); diff --git a/kernel/cpu.c b/kernel/cpu.c index 4e11e91010e1..1b6302ecbabe 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -330,6 +330,13 @@ void lockdep_assert_cpus_held(void) percpu_rwsem_assert_held(&cpu_hotplug_lock); } +#ifdef CONFIG_LOCKDEP +int lockdep_is_cpus_held(void) +{ + return percpu_rwsem_is_held(&cpu_hotplug_lock); +} +#endif + static void lockdep_acquire_cpus_lock(void) { rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_); -- cgit v1.2.3 From dcd42591ebb8a25895b551a5297ea9c24414ba54 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 13 Nov 2020 13:13:33 +0100 Subject: timer: Add timer_curr_running() This commit adds a timer_curr_running() function that verifies that the current code is running in the context of the specified timer's handler. Cc: Josh Triplett Cc: Steven Rostedt Cc: Mathieu Desnoyers Cc: Lai Jiangshan Cc: Joel Fernandes Cc: Neeraj Upadhyay Cc: Thomas Gleixner Tested-by: Boqun Feng Signed-off-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/timer.h | 2 ++ kernel/time/timer.c | 13 +++++++++++++ 2 files changed, 15 insertions(+) (limited to 'include/linux') diff --git a/include/linux/timer.h b/include/linux/timer.h index fda13c9d1256..4118a97e62fb 100644 --- a/include/linux/timer.h +++ b/include/linux/timer.h @@ -192,6 +192,8 @@ extern int try_to_del_timer_sync(struct timer_list *timer); #define del_singleshot_timer_sync(t) del_timer_sync(t) +extern bool timer_curr_running(struct timer_list *timer); + extern void init_timers(void); struct hrtimer; extern enum hrtimer_restart it_real_fn(struct hrtimer *); diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 8dbc008f8942..f9b2096456e5 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1237,6 +1237,19 @@ int try_to_del_timer_sync(struct timer_list *timer) } EXPORT_SYMBOL(try_to_del_timer_sync); +bool timer_curr_running(struct timer_list *timer) +{ + int i; + + for (i = 0; i < NR_BASES; i++) { + struct timer_base *base = this_cpu_ptr(&timer_bases[i]); + if (base->running_timer == timer) + return true; + } + + return false; +} + #ifdef CONFIG_PREEMPT_RT static __init void timer_base_init_expiry_lock(struct timer_base *base) { -- cgit v1.2.3 From ae19aaafae95a5487469433e9cae4c208f8d15cd Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 17 Nov 2020 11:30:18 -0800 Subject: torture: Add fuzzed hrtimer-based sleep functions This commit adds torture_hrtimeout_ns(), torture_hrtimeout_us(), torture_hrtimeout_ms(), torture_hrtimeout_jiffies(), and torture_hrtimeout_s(), each of which uses hrtimers to block for a fuzzed time interval. These functions are intended to be used by the various torture tests to decouple wakeups from the timer wheel, thus providing more opportunity for Murphy to insert destructive race conditions. Signed-off-by: Paul E. McKenney --- include/linux/torture.h | 7 +++++ kernel/torture.c | 75 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) (limited to 'include/linux') diff --git a/include/linux/torture.h b/include/linux/torture.h index 7f65bd1dd307..32941f8106b2 100644 --- a/include/linux/torture.h +++ b/include/linux/torture.h @@ -61,6 +61,13 @@ static inline void torture_random_init(struct torture_random_state *trsp) trsp->trs_count = 0; } +/* Definitions for high-resolution-timer sleeps. */ +int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_state *trsp); +int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp); +int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp); +int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp); +int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp); + /* Task shuffler, which causes CPUs to occasionally go idle. */ void torture_shuffle_task_register(struct task_struct *tp); int torture_shuffle_init(long shuffint); diff --git a/kernel/torture.c b/kernel/torture.c index 8562ac18d2eb..7548634e8a89 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -58,6 +58,81 @@ static int verbose; static int fullstop = FULLSTOP_RMMOD; static DEFINE_MUTEX(fullstop_mutex); +/* + * Schedule a high-resolution-timer sleep in nanoseconds, with a 32-bit + * nanosecond random fuzz. This function and its friends desynchronize + * testing from the timer wheel. + */ +int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_state *trsp) +{ + ktime_t hto = baset_ns; + + if (trsp) + hto += (torture_random(trsp) >> 3) % fuzzt_ns; + set_current_state(TASK_UNINTERRUPTIBLE); + return schedule_hrtimeout(&hto, HRTIMER_MODE_REL); +} +EXPORT_SYMBOL_GPL(torture_hrtimeout_ns); + +/* + * Schedule a high-resolution-timer sleep in microseconds, with a 32-bit + * nanosecond (not microsecond!) random fuzz. + */ +int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp) +{ + ktime_t baset_ns = baset_us * NSEC_PER_USEC; + + return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp); +} +EXPORT_SYMBOL_GPL(torture_hrtimeout_us); + +/* + * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit + * microsecond (not millisecond!) random fuzz. + */ +int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp) +{ + ktime_t baset_ns = baset_ms * NSEC_PER_MSEC; + u32 fuzzt_ns; + + if ((u32)~0U / NSEC_PER_USEC < fuzzt_us) + fuzzt_ns = (u32)~0U; + else + fuzzt_ns = fuzzt_us * NSEC_PER_USEC; + return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp); +} +EXPORT_SYMBOL_GPL(torture_hrtimeout_ms); + +/* + * Schedule a high-resolution-timer sleep in jiffies, with an + * implied one-jiffy random fuzz. This is intended to replace calls to + * schedule_timeout_interruptible() and friends. + */ +int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp) +{ + ktime_t baset_ns = jiffies_to_nsecs(baset_j); + + return torture_hrtimeout_ns(baset_ns, jiffies_to_nsecs(1), trsp); +} +EXPORT_SYMBOL_GPL(torture_hrtimeout_jiffies); + +/* + * Schedule a high-resolution-timer sleep in milliseconds, with a 32-bit + * millisecond (not second!) random fuzz. + */ +int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp) +{ + ktime_t baset_ns = baset_s * NSEC_PER_SEC; + u32 fuzzt_ns; + + if ((u32)~0U / NSEC_PER_MSEC < fuzzt_ms) + fuzzt_ns = (u32)~0U; + else + fuzzt_ns = fuzzt_ms * NSEC_PER_MSEC; + return torture_hrtimeout_ns(baset_ns, fuzzt_ns, trsp); +} +EXPORT_SYMBOL_GPL(torture_hrtimeout_s); + #ifdef CONFIG_HOTPLUG_CPU /* -- cgit v1.2.3 From 8a67a20bf257ca378d6e5588fbe4382966395ac8 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 25 Nov 2020 13:00:04 -0800 Subject: torture: Throttle VERBOSE_TOROUT_*() output This commit adds kernel boot parameters torture.verbose_sleep_frequency and torture.verbose_sleep_duration, which allow VERBOSE_TOROUT_*() output to be throttled with periodic sleeps on large systems. Signed-off-by: Paul E. McKenney --- Documentation/admin-guide/kernel-parameters.txt | 8 ++++++++ include/linux/torture.h | 15 +++++++++++++-- kernel/torture.c | 20 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 3244f9e04a64..18f3aa759e54 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5337,6 +5337,14 @@ are running concurrently, especially on systems with rotating-rust storage. + torture.verbose_sleep_frequency= [KNL] + Specifies how many verbose printk()s should be + emitted between each sleep. The default of zero + disables verbose-printk() sleeping. + + torture.verbose_sleep_duration= [KNL] + Duration of each verbose-printk() sleep in jiffies. + tp720= [HW,PS2] tpm_suspend_pcr=[HW,TPM] diff --git a/include/linux/torture.h b/include/linux/torture.h index 32941f8106b2..d62d13c8c69a 100644 --- a/include/linux/torture.h +++ b/include/linux/torture.h @@ -32,9 +32,20 @@ #define TOROUT_STRING(s) \ pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s) #define VERBOSE_TOROUT_STRING(s) \ - do { if (verbose) pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s); } while (0) +do { \ + if (verbose) { \ + verbose_torout_sleep(); \ + pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s); \ + } \ +} while (0) #define VERBOSE_TOROUT_ERRSTRING(s) \ - do { if (verbose) pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s); } while (0) +do { \ + if (verbose) { \ + verbose_torout_sleep(); \ + pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s); \ + } \ +} while (0) +void verbose_torout_sleep(void); /* Definitions for online/offline exerciser. */ typedef void torture_ofl_func(void); diff --git a/kernel/torture.c b/kernel/torture.c index 7ad5c9681580..93eeeb2b3d88 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -48,6 +48,12 @@ module_param(disable_onoff_at_boot, bool, 0444); static bool ftrace_dump_at_shutdown; module_param(ftrace_dump_at_shutdown, bool, 0444); +static int verbose_sleep_frequency; +module_param(verbose_sleep_frequency, int, 0444); + +static int verbose_sleep_duration = 1; +module_param(verbose_sleep_duration, int, 0444); + static char *torture_type; static int verbose; @@ -58,6 +64,20 @@ static int verbose; static int fullstop = FULLSTOP_RMMOD; static DEFINE_MUTEX(fullstop_mutex); +static atomic_t verbose_sleep_counter; + +/* + * Sleep if needed from VERBOSE_TOROUT*(). + */ +void verbose_torout_sleep(void) +{ + if (verbose_sleep_frequency > 0 && + verbose_sleep_duration > 0 && + !(atomic_inc_return(&verbose_sleep_counter) % verbose_sleep_frequency)) + schedule_timeout_uninterruptible(verbose_sleep_duration); +} +EXPORT_SYMBOL_GPL(verbose_torout_sleep); + /* * Schedule a high-resolution-timer sleep in nanoseconds, with a 32-bit * nanosecond random fuzz. This function and its friends desynchronize -- cgit v1.2.3 From 1afb95fee0342b8d9e05b0433e8e44a6dfd7c4a3 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sat, 19 Dec 2020 07:34:35 -0800 Subject: torture: Maintain torture-specific set of CPUs-online books The TREE01 rcutorture scenario intentionally creates confusion as to the number of available CPUs by specifying the "maxcpus=8 nr_cpus=43" kernel boot parameters. This can disable rcutorture's load shedding, which currently uses num_online_cpus(), which would count the extra 35 CPUs. However, the rcutorture guest OS will be provisioned with only 8 CPUs, which means that rcutorture will present full load even when all but one of the original 8 CPUs are offline. This can result in spurious errors due to extreme overloading of that single remaining CPU. This commit therefore keeps a separate set of books on the number of usable online CPUs, so that torture_num_online_cpus() is used for load shedding instead of num_online_cpus(). Note that initial sizing must use num_online_cpus() because torture_num_online_cpus() will return NR_CPUS until shortly after torture_onoff_init() is invoked. Reported-by: Frederic Weisbecker [ paulmck: Apply feedback from kernel test robot. ] Signed-off-by: Paul E. McKenney --- include/linux/torture.h | 5 +++++ kernel/rcu/rcutorture.c | 4 ++-- kernel/torture.c | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/torture.h b/include/linux/torture.h index d62d13c8c69a..0910c5803f35 100644 --- a/include/linux/torture.h +++ b/include/linux/torture.h @@ -48,6 +48,11 @@ do { \ void verbose_torout_sleep(void); /* Definitions for online/offline exerciser. */ +#ifdef CONFIG_HOTPLUG_CPU +int torture_num_online_cpus(void); +#else /* #ifdef CONFIG_HOTPLUG_CPU */ +static inline int torture_num_online_cpus(void) { return 1; } +#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */ typedef void torture_ofl_func(void); bool torture_offline(int cpu, long *n_onl_attempts, long *n_onl_successes, unsigned long *sum_offl, int *min_onl, int *max_onl); diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 76c838696366..a816df4e86e0 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1338,7 +1338,7 @@ static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp, struct torture_random_state *trsp) { unsigned long loops; - int noc = num_online_cpus(); + int noc = torture_num_online_cpus(); int rdrchked; int rdrchker; struct rcu_torture_reader_check *rtrcp; // Me. @@ -1658,7 +1658,7 @@ rcu_torture_reader(void *arg) torture_hrtimeout_us(500, 1000, &rand); lastsleep = jiffies + 10; } - while (num_online_cpus() < mynumonline && !torture_must_stop()) + while (torture_num_online_cpus() < mynumonline && !torture_must_stop()) schedule_timeout_interruptible(HZ / 5); stutter_wait("rcu_torture_reader"); } while (!torture_must_stop()); diff --git a/kernel/torture.c b/kernel/torture.c index 507a20be6950..01e336f1e5b2 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -175,6 +175,19 @@ static unsigned long sum_online; static int min_online = -1; static int max_online; +static int torture_online_cpus = NR_CPUS; + +/* + * Some torture testing leverages confusion as to the number of online + * CPUs. This function returns the torture-testing view of this number, + * which allows torture tests to load-balance appropriately. + */ +int torture_num_online_cpus(void) +{ + return READ_ONCE(torture_online_cpus); +} +EXPORT_SYMBOL_GPL(torture_num_online_cpus); + /* * Attempt to take a CPU offline. Return false if the CPU is already * offline or if it is not subject to CPU-hotplug operations. The @@ -229,6 +242,8 @@ bool torture_offline(int cpu, long *n_offl_attempts, long *n_offl_successes, *min_offl = delta; if (*max_offl < delta) *max_offl = delta; + WRITE_ONCE(torture_online_cpus, torture_online_cpus - 1); + WARN_ON_ONCE(torture_online_cpus <= 0); } return true; @@ -285,6 +300,7 @@ bool torture_online(int cpu, long *n_onl_attempts, long *n_onl_successes, *min_onl = delta; if (*max_onl < delta) *max_onl = delta; + WRITE_ONCE(torture_online_cpus, torture_online_cpus + 1); } return true; -- cgit v1.2.3 From 63f24a7fafd44899f001b8467b38fac5a534f63e Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Tue, 5 Jan 2021 13:02:28 +0100 Subject: vt: move set_leds to keyboard.c set_leds and compute_shiftstate are called from a single place in vt.c. Let's combine these two into vt_set_leds_compute_shiftstate. This allows for making keyboard_tasklet local in the next patch. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20210105120239.28031-1-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/keyboard.c | 11 ++++++++++- drivers/tty/vt/vt.c | 3 +-- include/linux/kbd_kern.h | 8 +------- 3 files changed, 12 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c index 52922d21a49f..32ec4242b1f2 100644 --- a/drivers/tty/vt/keyboard.c +++ b/drivers/tty/vt/keyboard.c @@ -372,6 +372,12 @@ static void to_utf8(struct vc_data *vc, uint c) } } +/* FIXME: review locking for vt.c callers */ +static void set_leds(void) +{ + tasklet_schedule(&keyboard_tasklet); +} + /* * Called after returning from RAW mode or when changing consoles - recompute * shift_down[] and shift_state from key_down[] maybe called when keymap is @@ -401,9 +407,12 @@ static void do_compute_shiftstate(void) } /* We still have to export this method to vt.c */ -void compute_shiftstate(void) +void vt_set_leds_compute_shiftstate(void) { unsigned long flags; + + set_leds(); + spin_lock_irqsave(&kbd_event_lock, flags); do_compute_shiftstate(); spin_unlock_irqrestore(&kbd_event_lock, flags); diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index d04a162939a4..fe4fedbc0386 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -1036,8 +1036,7 @@ void redraw_screen(struct vc_data *vc, int is_switch) } set_cursor(vc); if (is_switch) { - set_leds(); - compute_shiftstate(); + vt_set_leds_compute_shiftstate(); notify_update(vc); } } diff --git a/include/linux/kbd_kern.h b/include/linux/kbd_kern.h index 82f29aa35062..adf98004624b 100644 --- a/include/linux/kbd_kern.h +++ b/include/linux/kbd_kern.h @@ -71,12 +71,6 @@ extern void (*kbd_ledfunc)(unsigned int led); extern int set_console(int nr); extern void schedule_console_callback(void); -/* FIXME: review locking for vt.c callers */ -static inline void set_leds(void) -{ - tasklet_schedule(&keyboard_tasklet); -} - static inline int vc_kbd_mode(struct kbd_struct * kbd, int flag) { return ((kbd->modeflags >> flag) & 1); @@ -135,7 +129,7 @@ static inline void chg_vc_kbd_led(struct kbd_struct * kbd, int flag) struct console; -void compute_shiftstate(void); +void vt_set_leds_compute_shiftstate(void); /* defkeymap.c */ -- cgit v1.2.3 From a18a9da82c57804ce5977ea0e01b72ee3f40a51b Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Tue, 5 Jan 2021 13:02:29 +0100 Subject: vt: keyboard, make keyboard_tasklet local Now that the last extern user of the tasklet (set_leds) is in keyboard.c, we can make keyboard_tasklet local to this unit too. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20210105120239.28031-2-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/keyboard.c | 5 +++-- include/linux/kbd_kern.h | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c index 32ec4242b1f2..9f2eaa104ebc 100644 --- a/drivers/tty/vt/keyboard.c +++ b/drivers/tty/vt/keyboard.c @@ -131,6 +131,9 @@ static const unsigned char max_vals[] = { static const int NR_TYPES = ARRAY_SIZE(max_vals); +static void kbd_bh(unsigned long dummy); +static DECLARE_TASKLET_DISABLED_OLD(keyboard_tasklet, kbd_bh); + static struct input_handler kbd_handler; static DEFINE_SPINLOCK(kbd_event_lock); static DEFINE_SPINLOCK(led_lock); @@ -1258,8 +1261,6 @@ static void kbd_bh(unsigned long dummy) } } -DECLARE_TASKLET_DISABLED_OLD(keyboard_tasklet, kbd_bh); - #if defined(CONFIG_X86) || defined(CONFIG_IA64) || defined(CONFIG_ALPHA) ||\ defined(CONFIG_MIPS) || defined(CONFIG_PPC) || defined(CONFIG_SPARC) ||\ defined(CONFIG_PARISC) || defined(CONFIG_SUPERH) ||\ diff --git a/include/linux/kbd_kern.h b/include/linux/kbd_kern.h index adf98004624b..c40811d79769 100644 --- a/include/linux/kbd_kern.h +++ b/include/linux/kbd_kern.h @@ -6,8 +6,6 @@ #include #include -extern struct tasklet_struct keyboard_tasklet; - extern char *func_table[MAX_NR_FUNC]; /* -- cgit v1.2.3 From ff2047fb755d4415ec3c70ac799889371151796d Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Tue, 5 Jan 2021 13:02:35 +0100 Subject: vt: drop old FONT ioctls Drop support for these ioctls: * PIO_FONT, PIO_FONTX * GIO_FONT, GIO_FONTX * PIO_FONTRESET As was demonstrated by commit 90bfdeef83f1 (tty: make FONTX ioctl use the tty pointer they were actually passed), these ioctls are not used from userspace, as: 1) they used to be broken (set up font on current console, not the open one) and racy (before the commit above) 2) KDFONTOP ioctl is used for years instead Note that PIO_FONTRESET is defunct on most systems as VGA_CONSOLE is set on them for ages. That turns on BROKEN_GRAPHICS_PROGRAMS which makes PIO_FONTRESET just return an error. We are removing KD_FONT_FLAG_OLD here as it was used only by these removed ioctls. kd.h header exists both in kernel and uapi headers, so we can remove the kernel one completely. Everyone includeing kd.h will now automatically get the uapi one. There are now unused definitions of the ioctl numbers and "struct consolefontdesc" in kd.h, but as it is a uapi header, I am not touching these. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20210105120239.28031-8-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 39 +----------- drivers/tty/vt/vt_ioctl.c | 151 ---------------------------------------------- include/linux/kd.h | 8 --- 3 files changed, 3 insertions(+), 195 deletions(-) delete mode 100644 include/linux/kd.h (limited to 'include/linux') diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index fe4fedbc0386..284b07224c55 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -4583,16 +4583,8 @@ static int con_font_get(struct vc_data *vc, struct console_font_op *op) if (op->data && font.charcount > op->charcount) rc = -ENOSPC; - if (!(op->flags & KD_FONT_FLAG_OLD)) { - if (font.width > op->width || font.height > op->height) - rc = -ENOSPC; - } else { - if (font.width != 8) - rc = -EIO; - else if ((op->height && font.height > op->height) || - font.height > 32) - rc = -ENOSPC; - } + if (font.width > op->width || font.height > op->height) + rc = -ENOSPC; if (rc) goto out; @@ -4620,7 +4612,7 @@ static int con_font_set(struct vc_data *vc, struct console_font_op *op) return -EINVAL; if (op->charcount > 512) return -EINVAL; - if (op->width <= 0 || op->width > 32 || op->height > 32) + if (op->width <= 0 || op->width > 32 || !op->height || op->height > 32) return -EINVAL; size = (op->width+7)/8 * 32 * op->charcount; if (size > max_font_size) @@ -4630,31 +4622,6 @@ static int con_font_set(struct vc_data *vc, struct console_font_op *op) if (IS_ERR(font.data)) return PTR_ERR(font.data); - if (!op->height) { /* Need to guess font height [compat] */ - int h, i; - u8 *charmap = font.data; - - /* - * If from KDFONTOP ioctl, don't allow things which can be done - * in userland,so that we can get rid of this soon - */ - if (!(op->flags & KD_FONT_FLAG_OLD)) { - kfree(font.data); - return -EINVAL; - } - - for (h = 32; h > 0; h--) - for (i = 0; i < op->charcount; i++) - if (charmap[32*i+h-1]) - goto nonzero; - - kfree(font.data); - return -EINVAL; - - nonzero: - op->height = h; - } - font.charcount = op->charcount; font.width = op->width; font.height = op->height; diff --git a/drivers/tty/vt/vt_ioctl.c b/drivers/tty/vt/vt_ioctl.c index 3813c40f1b48..4a4cbd4a5f37 100644 --- a/drivers/tty/vt/vt_ioctl.c +++ b/drivers/tty/vt/vt_ioctl.c @@ -484,70 +484,6 @@ static int vt_k_ioctl(struct tty_struct *tty, unsigned int cmd, return 0; } -static inline int do_fontx_ioctl(struct vc_data *vc, int cmd, - struct consolefontdesc __user *user_cfd, - struct console_font_op *op) -{ - struct consolefontdesc cfdarg; - int i; - - if (copy_from_user(&cfdarg, user_cfd, sizeof(struct consolefontdesc))) - return -EFAULT; - - switch (cmd) { - case PIO_FONTX: - op->op = KD_FONT_OP_SET; - op->flags = KD_FONT_FLAG_OLD; - op->width = 8; - op->height = cfdarg.charheight; - op->charcount = cfdarg.charcount; - op->data = cfdarg.chardata; - return con_font_op(vc, op); - - case GIO_FONTX: - op->op = KD_FONT_OP_GET; - op->flags = KD_FONT_FLAG_OLD; - op->width = 8; - op->height = cfdarg.charheight; - op->charcount = cfdarg.charcount; - op->data = cfdarg.chardata; - i = con_font_op(vc, op); - if (i) - return i; - cfdarg.charheight = op->height; - cfdarg.charcount = op->charcount; - if (copy_to_user(user_cfd, &cfdarg, sizeof(struct consolefontdesc))) - return -EFAULT; - return 0; - } - return -EINVAL; -} - -static int vt_io_fontreset(struct vc_data *vc, struct console_font_op *op) -{ - int ret; - - if (__is_defined(BROKEN_GRAPHICS_PROGRAMS)) { - /* - * With BROKEN_GRAPHICS_PROGRAMS defined, the default font is - * not saved. - */ - return -ENOSYS; - } - - op->op = KD_FONT_OP_SET_DEFAULT; - op->data = NULL; - ret = con_font_op(vc, op); - if (ret) - return ret; - - console_lock(); - con_set_default_unimap(vc); - console_unlock(); - - return 0; -} - static inline int do_unimap_ioctl(int cmd, struct unimapdesc __user *user_ud, bool perm, struct vc_data *vc) { @@ -572,29 +508,7 @@ static inline int do_unimap_ioctl(int cmd, struct unimapdesc __user *user_ud, static int vt_io_ioctl(struct vc_data *vc, unsigned int cmd, void __user *up, bool perm) { - struct console_font_op op; /* used in multiple places here */ - switch (cmd) { - case PIO_FONT: - if (!perm) - return -EPERM; - op.op = KD_FONT_OP_SET; - op.flags = KD_FONT_FLAG_OLD | KD_FONT_FLAG_DONT_RECALC; /* Compatibility */ - op.width = 8; - op.height = 0; - op.charcount = 256; - op.data = up; - return con_font_op(vc, &op); - - case GIO_FONT: - op.op = KD_FONT_OP_GET; - op.flags = KD_FONT_FLAG_OLD; - op.width = 8; - op.height = 32; - op.charcount = 256; - op.data = up; - return con_font_op(vc, &op); - case PIO_CMAP: if (!perm) return -EPERM; @@ -603,20 +517,6 @@ static int vt_io_ioctl(struct vc_data *vc, unsigned int cmd, void __user *up, case GIO_CMAP: return con_get_cmap(up); - case PIO_FONTX: - if (!perm) - return -EPERM; - - fallthrough; - case GIO_FONTX: - return do_fontx_ioctl(vc, cmd, up, &op); - - case PIO_FONTRESET: - if (!perm) - return -EPERM; - - return vt_io_fontreset(vc, &op); - case PIO_SCRNMAP: if (!perm) return -EPERM; @@ -1059,54 +959,6 @@ void vc_SAK(struct work_struct *work) #ifdef CONFIG_COMPAT -struct compat_consolefontdesc { - unsigned short charcount; /* characters in font (256 or 512) */ - unsigned short charheight; /* scan lines per character (1-32) */ - compat_caddr_t chardata; /* font data in expanded form */ -}; - -static inline int -compat_fontx_ioctl(struct vc_data *vc, int cmd, - struct compat_consolefontdesc __user *user_cfd, - int perm, struct console_font_op *op) -{ - struct compat_consolefontdesc cfdarg; - int i; - - if (copy_from_user(&cfdarg, user_cfd, sizeof(struct compat_consolefontdesc))) - return -EFAULT; - - switch (cmd) { - case PIO_FONTX: - if (!perm) - return -EPERM; - op->op = KD_FONT_OP_SET; - op->flags = KD_FONT_FLAG_OLD; - op->width = 8; - op->height = cfdarg.charheight; - op->charcount = cfdarg.charcount; - op->data = compat_ptr(cfdarg.chardata); - return con_font_op(vc, op); - - case GIO_FONTX: - op->op = KD_FONT_OP_GET; - op->flags = KD_FONT_FLAG_OLD; - op->width = 8; - op->height = cfdarg.charheight; - op->charcount = cfdarg.charcount; - op->data = compat_ptr(cfdarg.chardata); - i = con_font_op(vc, op); - if (i) - return i; - cfdarg.charheight = op->height; - cfdarg.charcount = op->charcount; - if (copy_to_user(user_cfd, &cfdarg, sizeof(struct compat_consolefontdesc))) - return -EFAULT; - return 0; - } - return -EINVAL; -} - struct compat_console_font_op { compat_uint_t op; /* operation code KD_FONT_OP_* */ compat_uint_t flags; /* KD_FONT_FLAG_* */ @@ -1183,9 +1035,6 @@ long vt_compat_ioctl(struct tty_struct *tty, /* * these need special handlers for incompatible data structures */ - case PIO_FONTX: - case GIO_FONTX: - return compat_fontx_ioctl(vc, cmd, up, perm, &op); case KDFONTOP: return compat_kdfontop_ioctl(up, perm, &op, vc); diff --git a/include/linux/kd.h b/include/linux/kd.h deleted file mode 100644 index b130a18f860f..000000000000 --- a/include/linux/kd.h +++ /dev/null @@ -1,8 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LINUX_KD_H -#define _LINUX_KD_H - -#include - -#define KD_FONT_FLAG_OLD 0x80000000 /* Invoked via old interface [compat] */ -#endif /* _LINUX_KD_H */ -- cgit v1.2.3 From cac8a63063e33606c4b8419ef34ef7644d7c2fc4 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Tue, 5 Jan 2021 13:02:36 +0100 Subject: vgacon: drop BROKEN_GRAPHICS_PROGRAMS BROKEN_GRAPHICS_PROGRAMS is defined when CONFIG_VGA_CONSOLE=y. And vgacon.c is built exclusively in that case too. So the check for BROKEN_GRAPHICS_PROGRAMS is pointless in vgacon.c as it is always true. So remove the test and BROKEN_GRAPHICS_PROGRAMS completely. This also eliminates the need for vga_font_is_default global as it is only set and never read. Cc: dri-devel@lists.freedesktop.org Cc: linux-fbdev@vger.kernel.org Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20210105120239.28031-9-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/video/console/vgacon.c | 19 ------------------- include/linux/vt_kern.h | 12 ------------ 2 files changed, 31 deletions(-) (limited to 'include/linux') diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index 17876f0179b5..962c12be9774 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -90,7 +90,6 @@ static unsigned int vga_video_num_lines; /* Number of text lines */ static bool vga_can_do_color; /* Do we support colors? */ static unsigned int vga_default_font_height __read_mostly; /* Height of default screen font */ static unsigned char vga_video_type __read_mostly; /* Card type */ -static bool vga_font_is_default = true; static int vga_vesa_blanked; static bool vga_palette_blanked; static bool vga_is_gfx; @@ -878,7 +877,6 @@ static int vgacon_do_font_op(struct vgastate *state, char *arg, int set, beg = 0x0a; } -#ifdef BROKEN_GRAPHICS_PROGRAMS /* * All fonts are loaded in slot 0 (0:1 for 512 ch) */ @@ -886,24 +884,7 @@ static int vgacon_do_font_op(struct vgastate *state, char *arg, int set, if (!arg) return -EINVAL; /* Return to default font not supported */ - vga_font_is_default = false; font_select = ch512 ? 0x04 : 0x00; -#else - /* - * The default font is kept in slot 0 and is never touched. - * A custom font is loaded in slot 2 (256 ch) or 2:3 (512 ch) - */ - - if (set) { - vga_font_is_default = !arg; - if (!arg) - ch512 = false; /* Default font is always 256 */ - font_select = arg ? (ch512 ? 0x0e : 0x0a) : 0x00; - } - - if (!vga_font_is_default) - charmap += 4 * cmapsz; -#endif raw_spin_lock_irq(&vga_lock); /* First, the Sequencer */ diff --git a/include/linux/vt_kern.h b/include/linux/vt_kern.h index 349e39c3ab60..94e7a315479c 100644 --- a/include/linux/vt_kern.h +++ b/include/linux/vt_kern.h @@ -16,18 +16,6 @@ #include #include -/* - * Presently, a lot of graphics programs do not restore the contents of - * the higher font pages. Defining this flag will avoid use of them, but - * will lose support for PIO_FONTRESET. Note that many font operations are - * not likely to work with these programs anyway; they need to be - * fixed. The linux/Documentation directory includes a code snippet - * to save and restore the text font. - */ -#ifdef CONFIG_VGA_CONSOLE -#define BROKEN_GRAPHICS_PROGRAMS 1 -#endif - void kd_mksound(unsigned int hz, unsigned int ticks); int kbd_rate(struct kbd_repeat *rep); -- cgit v1.2.3 From 0bc1bd092af3c7c0b025ece93c3a86916f89f3ca Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Tue, 5 Jan 2021 13:02:38 +0100 Subject: tty_port: drop last traces of low_latency The main purpose of tty_port::low_latency was removed in commit a9c3f68f3cd8 (tty: Fix low_latency BUG) back in 2014. It was left in place for drivers as an optional tune knob. But only one driver has been using it until the previous commit. So remove this misconcept completely, given there are no users. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20210105120239.28031-11-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- Documentation/networking/caif/caif.rst | 1 - drivers/char/pcmcia/synclink_cs.c | 2 -- drivers/net/caif/caif_serial.c | 3 +-- drivers/s390/char/con3215.c | 1 - drivers/s390/char/sclp_tty.c | 1 - drivers/s390/char/sclp_vt220.c | 1 - drivers/s390/char/tty3270.c | 2 -- drivers/tty/amiserial.c | 3 --- drivers/tty/hvc/hvcs.c | 2 +- drivers/tty/ipwireless/tty.c | 1 - drivers/tty/mxser.c | 1 - drivers/tty/serial/ifx6x60.c | 3 --- drivers/tty/serial/max3100.c | 3 --- drivers/tty/serial/serial_core.c | 3 --- drivers/tty/synclink_gt.c | 1 - include/linux/tty.h | 3 +-- 16 files changed, 3 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/caif/caif.rst b/Documentation/networking/caif/caif.rst index a07213030ccf..81a14373d780 100644 --- a/Documentation/networking/caif/caif.rst +++ b/Documentation/networking/caif/caif.rst @@ -68,7 +68,6 @@ There are debugfs parameters provided for serial communication. * tty_status: Prints the bit-mask tty status information - 0x01 - tty->warned is on. - - 0x02 - tty->low_latency is on. - 0x04 - tty->packed is on. - 0x08 - tty->flow_stopped is on. - 0x10 - tty->hw_stopped is on. diff --git a/drivers/char/pcmcia/synclink_cs.c b/drivers/char/pcmcia/synclink_cs.c index e342daa73d1b..2be8d9a8eec5 100644 --- a/drivers/char/pcmcia/synclink_cs.c +++ b/drivers/char/pcmcia/synclink_cs.c @@ -2494,8 +2494,6 @@ static int mgslpc_open(struct tty_struct *tty, struct file * filp) printk("%s(%d):mgslpc_open(%s), old ref count = %d\n", __FILE__, __LINE__, tty->driver->name, port->count); - port->low_latency = (port->flags & ASYNC_LOW_LATENCY) ? 1 : 0; - spin_lock_irqsave(&info->netlock, flags); if (info->netcount) { retval = -EBUSY; diff --git a/drivers/net/caif/caif_serial.c b/drivers/net/caif/caif_serial.c index bcc14c5875bf..8215cd77301f 100644 --- a/drivers/net/caif/caif_serial.c +++ b/drivers/net/caif/caif_serial.c @@ -89,8 +89,7 @@ static inline void update_tty_status(struct ser_device *ser) ser->tty_status = ser->tty->stopped << 5 | ser->tty->flow_stopped << 3 | - ser->tty->packet << 2 | - ser->tty->port->low_latency << 1; + ser->tty->packet << 2; } static inline void debugfs_init(struct ser_device *ser, struct tty_struct *tty) { diff --git a/drivers/s390/char/con3215.c b/drivers/s390/char/con3215.c index 1354c42d95aa..671efee612af 100644 --- a/drivers/s390/char/con3215.c +++ b/drivers/s390/char/con3215.c @@ -914,7 +914,6 @@ static int tty3215_open(struct tty_struct *tty, struct file * filp) tty_port_tty_set(&raw->port, tty); - raw->port.low_latency = 0; /* don't use bottom half for pushing chars */ /* * Start up 3215 device */ diff --git a/drivers/s390/char/sclp_tty.c b/drivers/s390/char/sclp_tty.c index 5aff8b684eb2..013bcc331305 100644 --- a/drivers/s390/char/sclp_tty.c +++ b/drivers/s390/char/sclp_tty.c @@ -65,7 +65,6 @@ sclp_tty_open(struct tty_struct *tty, struct file *filp) { tty_port_tty_set(&sclp_port, tty); tty->driver_data = NULL; - sclp_port.low_latency = 0; return 0; } diff --git a/drivers/s390/char/sclp_vt220.c b/drivers/s390/char/sclp_vt220.c index 3f9a6ef650fa..047f812d1a1c 100644 --- a/drivers/s390/char/sclp_vt220.c +++ b/drivers/s390/char/sclp_vt220.c @@ -560,7 +560,6 @@ sclp_vt220_open(struct tty_struct *tty, struct file *filp) { if (tty->count == 1) { tty_port_tty_set(&sclp_vt220_port, tty); - sclp_vt220_port.low_latency = 0; if (!tty->winsize.ws_row && !tty->winsize.ws_col) { tty->winsize.ws_row = 24; tty->winsize.ws_col = 80; diff --git a/drivers/s390/char/tty3270.c b/drivers/s390/char/tty3270.c index aec996de44d9..15692449a1c3 100644 --- a/drivers/s390/char/tty3270.c +++ b/drivers/s390/char/tty3270.c @@ -967,7 +967,6 @@ static int tty3270_install(struct tty_driver *driver, struct tty_struct *tty) tty->driver_data = tp; tty->winsize.ws_row = tp->view.rows - 2; tty->winsize.ws_col = tp->view.cols; - tp->port.low_latency = 0; tp->inattr = TF_INPUT; goto port_install; } @@ -996,7 +995,6 @@ static int tty3270_install(struct tty_driver *driver, struct tty_struct *tty) return rc; } - tp->port.low_latency = 0; tty->winsize.ws_row = tp->view.rows - 2; tty->winsize.ws_col = tp->view.cols; diff --git a/drivers/tty/amiserial.c b/drivers/tty/amiserial.c index 13f63c01c589..18b78ea110ef 100644 --- a/drivers/tty/amiserial.c +++ b/drivers/tty/amiserial.c @@ -998,7 +998,6 @@ static int set_serial_info(struct tty_struct *tty, struct serial_struct *ss) state->custom_divisor = ss->custom_divisor; port->close_delay = ss->close_delay * HZ/100; port->closing_wait = ss->closing_wait * HZ/100; - port->low_latency = (port->flags & ASYNC_LOW_LATENCY) ? 1 : 0; check_and_exit: if (tty_port_initialized(port)) { @@ -1386,8 +1385,6 @@ static int rs_open(struct tty_struct *tty, struct file * filp) tty->driver_data = info; tty->port = port; - port->low_latency = (port->flags & ASYNC_LOW_LATENCY) ? 1 : 0; - retval = startup(tty, info); if (retval) { return retval; diff --git a/drivers/tty/hvc/hvcs.c b/drivers/tty/hvc/hvcs.c index 509d1042825a..dfe02283ed23 100644 --- a/drivers/tty/hvc/hvcs.c +++ b/drivers/tty/hvc/hvcs.c @@ -605,7 +605,7 @@ static int hvcs_io(struct hvcs_struct *hvcsd) hvcsd->todo_mask |= HVCS_QUICK_READ; spin_unlock_irqrestore(&hvcsd->lock, flags); - /* This is synch because tty->low_latency == 1 */ + /* This is synch -- FIXME :js: it is not! */ if(got) tty_flip_buffer_push(&hvcsd->port); diff --git a/drivers/tty/ipwireless/tty.c b/drivers/tty/ipwireless/tty.c index 23584769fc29..6dacbc5e286c 100644 --- a/drivers/tty/ipwireless/tty.c +++ b/drivers/tty/ipwireless/tty.c @@ -101,7 +101,6 @@ static int ipw_open(struct tty_struct *linux_tty, struct file *filp) tty->port.tty = linux_tty; linux_tty->driver_data = tty; - tty->port.low_latency = 1; if (tty->tty_type == TTYTYPE_MODEM) ipwireless_ppp_open(tty->network); diff --git a/drivers/tty/mxser.c b/drivers/tty/mxser.c index 3703987c4666..4203b64bccdb 100644 --- a/drivers/tty/mxser.c +++ b/drivers/tty/mxser.c @@ -1273,7 +1273,6 @@ static int mxser_set_serial_info(struct tty_struct *tty, (ss->flags & ASYNC_FLAGS)); port->close_delay = ss->close_delay * HZ / 100; port->closing_wait = ss->closing_wait * HZ / 100; - port->low_latency = (port->flags & ASYNC_LOW_LATENCY) ? 1 : 0; if ((port->flags & ASYNC_SPD_MASK) == ASYNC_SPD_CUST && (ss->baud_base != info->baud_base || ss->custom_divisor != diff --git a/drivers/tty/serial/ifx6x60.c b/drivers/tty/serial/ifx6x60.c index 182e0ccd60b2..d4ef88ee22d0 100644 --- a/drivers/tty/serial/ifx6x60.c +++ b/drivers/tty/serial/ifx6x60.c @@ -565,9 +565,6 @@ static int ifx_port_activate(struct tty_port *port, struct tty_struct *tty) /* put port data into this tty */ tty->driver_data = ifx_dev; - /* allows flip string push from int context */ - port->low_latency = 1; - /* set flag to allows data transfer */ set_bit(IFX_SPI_STATE_IO_AVAILABLE, &ifx_dev->flags); diff --git a/drivers/tty/serial/max3100.c b/drivers/tty/serial/max3100.c index 371569a0fd00..3c92d4e01488 100644 --- a/drivers/tty/serial/max3100.c +++ b/drivers/tty/serial/max3100.c @@ -521,9 +521,6 @@ max3100_set_termios(struct uart_port *port, struct ktermios *termios, MAX3100_STATUS_PE | MAX3100_STATUS_FE | MAX3100_STATUS_OE; - /* we are sending char from a workqueue so enable */ - s->port.state->port.low_latency = 1; - if (s->poll_time > 0) del_timer_sync(&s->timer); diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 828f9ad1be49..7dacdb6a8534 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -975,7 +975,6 @@ static int uart_set_info(struct tty_struct *tty, struct tty_port *port, port->closing_wait = closing_wait; if (new_info->xmit_fifo_size) uport->fifosize = new_info->xmit_fifo_size; - port->low_latency = (uport->flags & UPF_LOW_LATENCY) ? 1 : 0; check_and_exit: retval = 0; @@ -1795,8 +1794,6 @@ static int uart_port_activate(struct tty_port *port, struct tty_struct *tty) if (!uport || uport->flags & UPF_DEAD) return -ENXIO; - port->low_latency = (uport->flags & UPF_LOW_LATENCY) ? 1 : 0; - /* * Start up the serial port. */ diff --git a/drivers/tty/synclink_gt.c b/drivers/tty/synclink_gt.c index c0b384e3ed4d..644173786bf0 100644 --- a/drivers/tty/synclink_gt.c +++ b/drivers/tty/synclink_gt.c @@ -672,7 +672,6 @@ static int open(struct tty_struct *tty, struct file *filp) DBGINFO(("%s open, old ref count = %d\n", info->device_name, info->port.count)); mutex_lock(&info->port.mutex); - info->port.low_latency = (info->port.flags & ASYNC_LOW_LATENCY) ? 1 : 0; spin_lock_irqsave(&info->netlock, flags); if (info->netcount) { diff --git a/include/linux/tty.h b/include/linux/tty.h index 12be8b16cdef..b57f6812b3ba 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -240,8 +240,7 @@ struct tty_port { wait_queue_head_t delta_msr_wait; /* Modem status change */ unsigned long flags; /* User TTY flags ASYNC_ */ unsigned long iflags; /* Internal flags TTY_PORT_ */ - unsigned char console:1, /* port is a console */ - low_latency:1; /* optional: tune for latency */ + unsigned char console:1; /* port is a console */ struct mutex mutex; /* Locking */ struct mutex buf_mutex; /* Buffer alloc lock */ unsigned char *xmit_buf; /* Optional buffer */ -- cgit v1.2.3 From f446776ebffb9d3fb145b8d84f7f358f64cb54d6 Mon Sep 17 00:00:00 2001 From: Corey Minyard Date: Mon, 23 Nov 2020 18:49:01 -0600 Subject: tty: Export redirect release This will be required by the pty code when it removes tty_vhangup() on master close. Signed-off-by: Corey Minyard Link: https://lore.kernel.org/r/20201124004902.1398477-2-minyard@acm.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/tty_io.c | 32 ++++++++++++++++++++++++-------- include/linux/tty.h | 1 + 2 files changed, 25 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 95ba028ef668..225f4933fbde 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -540,6 +540,28 @@ void tty_wakeup(struct tty_struct *tty) EXPORT_SYMBOL_GPL(tty_wakeup); +/** + * tty_release_redirect - Release a redirect on a pty if present + * @tty: tty device + * + * This is available to the pty code so if the master closes, if the + * slave is a redirect it can release the redirect. + */ +struct file *tty_release_redirect(struct tty_struct *tty) +{ + struct file *f = NULL; + + spin_lock(&redirect_lock); + if (redirect && file_tty(redirect) == tty) { + f = redirect; + redirect = NULL; + } + spin_unlock(&redirect_lock); + + return f; +} +EXPORT_SYMBOL_GPL(tty_release_redirect); + /** * __tty_hangup - actual handler for hangup events * @tty: tty device @@ -566,7 +588,7 @@ EXPORT_SYMBOL_GPL(tty_wakeup); static void __tty_hangup(struct tty_struct *tty, int exit_session) { struct file *cons_filp = NULL; - struct file *filp, *f = NULL; + struct file *filp, *f; struct tty_file_private *priv; int closecount = 0, n; int refs; @@ -574,13 +596,7 @@ static void __tty_hangup(struct tty_struct *tty, int exit_session) if (!tty) return; - - spin_lock(&redirect_lock); - if (redirect && file_tty(redirect) == tty) { - f = redirect; - redirect = NULL; - } - spin_unlock(&redirect_lock); + f = tty_release_redirect(tty); tty_lock(tty); diff --git a/include/linux/tty.h b/include/linux/tty.h index b57f6812b3ba..dd6ded6138f7 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -421,6 +421,7 @@ extern void tty_kclose(struct tty_struct *tty); extern int tty_dev_name_to_number(const char *name, dev_t *number); extern int tty_ldisc_lock(struct tty_struct *tty, unsigned long timeout); extern void tty_ldisc_unlock(struct tty_struct *tty); +extern struct file *tty_release_redirect(struct tty_struct *tty); #else static inline void tty_kref_put(struct tty_struct *tty) { } -- cgit v1.2.3 From 30bfce109420912f201d4f295f9130ff44f04b41 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 6 Jan 2021 13:06:36 -0800 Subject: net: remove ndo_udp_tunnel_* callbacks All UDP tunnel port management is now routed via udp_tunnel_nic infra directly. Remove the old callbacks. Reviewed-by: Alexander Duyck Reviewed-by: Jacob Keller Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/amd/xgbe/xgbe-drv.c | 2 -- drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c | 2 -- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 2 -- drivers/net/ethernet/cavium/liquidio/lio_main.c | 2 -- drivers/net/ethernet/cavium/liquidio/lio_vf_main.c | 2 -- drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c | 2 -- drivers/net/ethernet/cisco/enic/enic_main.c | 4 ---- drivers/net/ethernet/emulex/benet/be_main.c | 2 -- drivers/net/ethernet/intel/fm10k/fm10k_netdev.c | 2 -- drivers/net/ethernet/intel/i40e/i40e_main.c | 2 -- drivers/net/ethernet/intel/ice/ice_main.c | 2 -- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 2 -- drivers/net/ethernet/mellanox/mlx4/en_netdev.c | 4 ---- drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 2 -- drivers/net/ethernet/mellanox/mlx5/core/en_rep.c | 2 -- drivers/net/ethernet/netronome/nfp/nfp_net_common.c | 2 -- drivers/net/ethernet/qlogic/qede/qede_main.c | 6 ------ drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c | 2 -- drivers/net/ethernet/sfc/efx.c | 2 -- drivers/net/netdevsim/netdev.c | 2 -- include/linux/netdevice.h | 17 ----------------- 21 files changed, 65 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index 2709a2db5657..99b6d5a9f1d9 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -2295,8 +2295,6 @@ static const struct net_device_ops xgbe_netdev_ops = { .ndo_setup_tc = xgbe_setup_tc, .ndo_fix_features = xgbe_fix_features, .ndo_set_features = xgbe_set_features, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = xgbe_features_check, }; diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c index 28069b290862..b652ed72a621 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c @@ -13071,8 +13071,6 @@ static const struct net_device_ops bnx2x_netdev_ops = { .ndo_get_phys_port_id = bnx2x_get_phys_port_id, .ndo_set_vf_link_state = bnx2x_set_vf_link_state, .ndo_features_check = bnx2x_features_check, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, }; static int bnx2x_set_coherency_mask(struct bnx2x *bp) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index bbd2a07dc329..d31a5ad7522a 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -12091,8 +12091,6 @@ static const struct net_device_ops bnxt_netdev_ops = { #ifdef CONFIG_RFS_ACCEL .ndo_rx_flow_steer = bnxt_rx_flow_steer, #endif - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_bpf = bnxt_xdp, .ndo_xdp_xmit = bnxt_xdp_xmit, .ndo_bridge_getlink = bnxt_bridge_getlink, diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c index 7d00d3a8ded4..7c5af4beedc6 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c @@ -3219,8 +3219,6 @@ static const struct net_device_ops lionetdevops = { .ndo_do_ioctl = liquidio_ioctl, .ndo_fix_features = liquidio_fix_features, .ndo_set_features = liquidio_set_features, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_set_vf_mac = liquidio_set_vf_mac, .ndo_set_vf_vlan = liquidio_set_vf_vlan, .ndo_get_vf_config = liquidio_get_vf_config, diff --git a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c index 103440f97bc8..516f166ceff8 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c @@ -1879,8 +1879,6 @@ static const struct net_device_ops lionetdevops = { .ndo_do_ioctl = liquidio_ioctl, .ndo_fix_features = liquidio_fix_features, .ndo_set_features = liquidio_set_features, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, }; static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf) diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c index 7fd264a6d085..15542661e3d2 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -3882,8 +3882,6 @@ static const struct net_device_ops cxgb4_netdev_ops = { #endif /* CONFIG_CHELSIO_T4_FCOE */ .ndo_set_tx_maxrate = cxgb_set_tx_maxrate, .ndo_setup_tc = cxgb_setup_tc, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = cxgb_features_check, .ndo_fix_features = cxgb_fix_features, }; diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c index fb269d587b74..f04ec53544ae 100644 --- a/drivers/net/ethernet/cisco/enic/enic_main.c +++ b/drivers/net/ethernet/cisco/enic/enic_main.c @@ -2509,8 +2509,6 @@ static const struct net_device_ops enic_netdev_dynamic_ops = { #ifdef CONFIG_RFS_ACCEL .ndo_rx_flow_steer = enic_rx_flow_steer, #endif - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = enic_features_check, }; @@ -2535,8 +2533,6 @@ static const struct net_device_ops enic_netdev_ops = { #ifdef CONFIG_RFS_ACCEL .ndo_rx_flow_steer = enic_rx_flow_steer, #endif - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = enic_features_check, }; diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c index d402d83d9edd..b6eba29d8e99 100644 --- a/drivers/net/ethernet/emulex/benet/be_main.c +++ b/drivers/net/ethernet/emulex/benet/be_main.c @@ -5179,8 +5179,6 @@ static const struct net_device_ops be_netdev_ops = { #endif .ndo_bridge_setlink = be_ndo_bridge_setlink, .ndo_bridge_getlink = be_ndo_bridge_getlink, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = be_features_check, .ndo_get_phys_port_id = be_get_phys_port_id, }; diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c index 5c19ff452558..2fb52bd6fc0e 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c @@ -1531,8 +1531,6 @@ static const struct net_device_ops fm10k_netdev_ops = { .ndo_set_vf_rate = fm10k_ndo_set_vf_bw, .ndo_get_vf_config = fm10k_ndo_get_vf_config, .ndo_get_vf_stats = fm10k_ndo_get_vf_stats, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_dfwd_add_station = fm10k_dfwd_add_station, .ndo_dfwd_del_station = fm10k_dfwd_del_station, .ndo_features_check = fm10k_features_check, diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 1db482d310c2..521ea9df38d5 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -12804,8 +12804,6 @@ static const struct net_device_ops i40e_netdev_ops = { .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, .ndo_set_vf_trust = i40e_ndo_set_vf_trust, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_get_phys_port_id = i40e_get_phys_port_id, .ndo_fdb_add = i40e_ndo_fdb_add, .ndo_features_check = i40e_features_check, diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index c52b9bb0e3ab..6e251dfffc91 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -6790,6 +6790,4 @@ static const struct net_device_ops ice_netdev_ops = { .ndo_bpf = ice_xdp, .ndo_xdp_xmit = ice_xdp_xmit, .ndo_xsk_wakeup = ice_xsk_wakeup, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, }; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 393d1c2cd853..6cbbe09ce8a0 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -10278,8 +10278,6 @@ static const struct net_device_ops ixgbe_netdev_ops = { .ndo_bridge_getlink = ixgbe_ndo_bridge_getlink, .ndo_dfwd_add_station = ixgbe_fwd_add, .ndo_dfwd_del_station = ixgbe_fwd_del, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = ixgbe_features_check, .ndo_bpf = ixgbe_xdp, .ndo_xdp_xmit = ixgbe_xdp_xmit, diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index 32aad4d32b88..51b9700fce83 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -2839,8 +2839,6 @@ static const struct net_device_ops mlx4_netdev_ops = { .ndo_rx_flow_steer = mlx4_en_filter_rfs, #endif .ndo_get_phys_port_id = mlx4_en_get_phys_port_id, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = mlx4_en_features_check, .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, .ndo_bpf = mlx4_xdp, @@ -2873,8 +2871,6 @@ static const struct net_device_ops mlx4_netdev_ops_master = { .ndo_rx_flow_steer = mlx4_en_filter_rfs, #endif .ndo_get_phys_port_id = mlx4_en_get_phys_port_id, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = mlx4_en_features_check, .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, .ndo_bpf = mlx4_xdp, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 7a79d330c075..f27f509ab028 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -4621,8 +4621,6 @@ const struct net_device_ops mlx5e_netdev_ops = { .ndo_change_mtu = mlx5e_change_nic_mtu, .ndo_do_ioctl = mlx5e_ioctl, .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = mlx5e_features_check, .ndo_tx_timeout = mlx5e_tx_timeout, .ndo_bpf = mlx5e_xdp, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 989c70c1eda3..cfa0e8552975 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -653,8 +653,6 @@ static const struct net_device_ops mlx5e_netdev_ops_uplink_rep = { .ndo_has_offload_stats = mlx5e_rep_has_offload_stats, .ndo_get_offload_stats = mlx5e_rep_get_offload_stats, .ndo_change_mtu = mlx5e_uplink_rep_change_mtu, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = mlx5e_features_check, .ndo_set_vf_mac = mlx5e_set_vf_mac, .ndo_set_vf_rate = mlx5e_set_vf_rate, diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index f21fb573ea3e..7ba8f4c7f26d 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -3656,8 +3656,6 @@ const struct net_device_ops nfp_net_netdev_ops = { .ndo_set_features = nfp_net_set_features, .ndo_features_check = nfp_net_features_check, .ndo_get_phys_port_name = nfp_net_get_phys_port_name, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_bpf = nfp_net_xdp, .ndo_get_devlink_port = nfp_devlink_get_devlink_port, }; diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index 9cf960a6d007..4bf94797aac5 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -663,8 +663,6 @@ static const struct net_device_ops qede_netdev_ops = { .ndo_get_vf_config = qede_get_vf_config, .ndo_set_vf_rate = qede_set_vf_rate, #endif - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = qede_features_check, .ndo_bpf = qede_xdp, #ifdef CONFIG_RFS_ACCEL @@ -688,8 +686,6 @@ static const struct net_device_ops qede_netdev_vf_ops = { .ndo_fix_features = qede_fix_features, .ndo_set_features = qede_set_features, .ndo_get_stats64 = qede_get_stats64, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = qede_features_check, }; @@ -707,8 +703,6 @@ static const struct net_device_ops qede_netdev_vf_xdp_ops = { .ndo_fix_features = qede_fix_features, .ndo_set_features = qede_set_features, .ndo_get_stats64 = qede_get_stats64, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = qede_features_check, .ndo_bpf = qede_xdp, .ndo_xdp_xmit = qede_xdp_transmit, diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c index c2faf96fcade..96b947fde646 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c @@ -520,8 +520,6 @@ static const struct net_device_ops qlcnic_netdev_ops = { .ndo_fdb_del = qlcnic_fdb_del, .ndo_fdb_dump = qlcnic_fdb_dump, .ndo_get_phys_port_id = qlcnic_get_phys_port_id, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_features_check = qlcnic_features_check, #ifdef CONFIG_QLCNIC_SRIOV .ndo_set_vf_mac = qlcnic_sriov_set_vf_mac, diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 718308076341..36c8625a6fd7 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -612,8 +612,6 @@ static const struct net_device_ops efx_netdev_ops = { #ifdef CONFIG_RFS_ACCEL .ndo_rx_flow_steer = efx_filter_rfs, #endif - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_xdp_xmit = efx_xdp_xmit, .ndo_bpf = efx_xdp }; diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c index 7178468302c8..aec92440eef1 100644 --- a/drivers/net/netdevsim/netdev.c +++ b/drivers/net/netdevsim/netdev.c @@ -258,8 +258,6 @@ static const struct net_device_ops nsim_netdev_ops = { .ndo_setup_tc = nsim_setup_tc, .ndo_set_features = nsim_set_features, .ndo_bpf = nsim_bpf, - .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, - .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, .ndo_get_devlink_port = nsim_get_devlink_port, }; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 259be67644e3..1ec3ac5d5bbf 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1213,19 +1213,6 @@ struct netdev_net_notifier { * struct netdev_phys_item_id *ppid) * Called to get the parent ID of the physical port of this device. * - * void (*ndo_udp_tunnel_add)(struct net_device *dev, - * struct udp_tunnel_info *ti); - * Called by UDP tunnel to notify a driver about the UDP port and socket - * address family that a UDP tunnel is listnening to. It is called only - * when a new port starts listening. The operation is protected by the - * RTNL. - * - * void (*ndo_udp_tunnel_del)(struct net_device *dev, - * struct udp_tunnel_info *ti); - * Called by UDP tunnel to notify the driver about a UDP port and socket - * address family that the UDP tunnel is not listening to anymore. The - * operation is protected by the RTNL. - * * void* (*ndo_dfwd_add_station)(struct net_device *pdev, * struct net_device *dev) * Called by upper layer devices to accelerate switching or other @@ -1464,10 +1451,6 @@ struct net_device_ops { struct netdev_phys_item_id *ppid); int (*ndo_get_phys_port_name)(struct net_device *dev, char *name, size_t len); - void (*ndo_udp_tunnel_add)(struct net_device *dev, - struct udp_tunnel_info *ti); - void (*ndo_udp_tunnel_del)(struct net_device *dev, - struct udp_tunnel_info *ti); void* (*ndo_dfwd_add_station)(struct net_device *pdev, struct net_device *dev); void (*ndo_dfwd_del_station)(struct net_device *pdev, -- cgit v1.2.3 From 8b86850bf9ef259e194ce49c776aded3b8c281fb Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Wed, 6 Jan 2021 09:09:44 -0800 Subject: net: phy: bcm7xxx: Add an entry for BCM72116 BCM72116 features a 28nm integrated EPHY, add an entry to match this PHY OUI. Signed-off-by: Florian Fainelli Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/r/20210106170944.1253046-1-f.fainelli@gmail.com Signed-off-by: Jakub Kicinski --- drivers/net/phy/bcm7xxx.c | 2 ++ include/linux/brcmphy.h | 1 + 2 files changed, 3 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/bcm7xxx.c b/drivers/net/phy/bcm7xxx.c index 15812001b3ff..e79297a4bae8 100644 --- a/drivers/net/phy/bcm7xxx.c +++ b/drivers/net/phy/bcm7xxx.c @@ -612,6 +612,7 @@ static void bcm7xxx_28nm_remove(struct phy_device *phydev) static struct phy_driver bcm7xxx_driver[] = { BCM7XXX_28NM_EPHY(PHY_ID_BCM72113, "Broadcom BCM72113"), + BCM7XXX_28NM_EPHY(PHY_ID_BCM72116, "Broadcom BCM72116"), BCM7XXX_28NM_GPHY(PHY_ID_BCM7250, "Broadcom BCM7250"), BCM7XXX_28NM_EPHY(PHY_ID_BCM7255, "Broadcom BCM7255"), BCM7XXX_28NM_EPHY(PHY_ID_BCM7260, "Broadcom BCM7260"), @@ -633,6 +634,7 @@ static struct phy_driver bcm7xxx_driver[] = { static struct mdio_device_id __maybe_unused bcm7xxx_tbl[] = { { PHY_ID_BCM72113, 0xfffffff0 }, + { PHY_ID_BCM72116, 0xfffffff0, }, { PHY_ID_BCM7250, 0xfffffff0, }, { PHY_ID_BCM7255, 0xfffffff0, }, { PHY_ID_BCM7260, 0xfffffff0, }, diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index d0bd226d6bd9..de9430d55c90 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -31,6 +31,7 @@ #define PHY_ID_BCM89610 0x03625cd0 #define PHY_ID_BCM72113 0x35905310 +#define PHY_ID_BCM72116 0x35905350 #define PHY_ID_BCM7250 0xae025280 #define PHY_ID_BCM7255 0xae025120 #define PHY_ID_BCM7260 0xae025190 -- cgit v1.2.3 From f46b9b8ee89b52f6ee1f4da49d392e609746ab10 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Thu, 7 Jan 2021 03:24:00 +0200 Subject: net: dsa: move the Broadcom tag information in a separate header file It is a bit strange to see something as specific as Broadcom SYSTEMPORT bits in the main DSA include file. Move these away into a separate header, and have the tagger and the SYSTEMPORT driver include them. Signed-off-by: Vladimir Oltean Acked-by: Florian Fainelli Signed-off-by: Jakub Kicinski --- MAINTAINERS | 1 + drivers/net/ethernet/broadcom/bcmsysport.c | 1 + include/linux/dsa/brcm.h | 16 ++++++++++++++++ include/net/dsa.h | 6 ------ net/dsa/tag_brcm.c | 1 + 5 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 include/linux/dsa/brcm.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index 7c1e45c416b1..3854aca806f5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3400,6 +3400,7 @@ L: openwrt-devel@lists.openwrt.org (subscribers-only) S: Supported F: Documentation/devicetree/bindings/net/dsa/brcm,b53.yaml F: drivers/net/dsa/b53/* +F: include/linux/dsa/brcm.h F: include/linux/platform_data/b53.h BROADCOM BCM2711/BCM2835 ARM ARCHITECTURE diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c index b1ae9eb8f247..9ef743d6ba67 100644 --- a/drivers/net/ethernet/broadcom/bcmsysport.c +++ b/drivers/net/ethernet/broadcom/bcmsysport.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/include/linux/dsa/brcm.h b/include/linux/dsa/brcm.h new file mode 100644 index 000000000000..47545a948784 --- /dev/null +++ b/include/linux/dsa/brcm.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2014 Broadcom Corporation + */ + +/* Included by drivers/net/ethernet/broadcom/bcmsysport.c and + * net/dsa/tag_brcm.c + */ +#ifndef _NET_DSA_BRCM_H +#define _NET_DSA_BRCM_H + +/* Broadcom tag specific helpers to insert and extract queue/port number */ +#define BRCM_TAG_SET_PORT_QUEUE(p, q) ((p) << 8 | q) +#define BRCM_TAG_GET_PORT(v) ((v) >> 8) +#define BRCM_TAG_GET_QUEUE(v) ((v) & 0xff) + +#endif diff --git a/include/net/dsa.h b/include/net/dsa.h index 6b74690bd8d4..e92a09c77f96 100644 --- a/include/net/dsa.h +++ b/include/net/dsa.h @@ -878,12 +878,6 @@ static inline int call_dsa_notifiers(unsigned long val, struct net_device *dev, } #endif -/* Broadcom tag specific helpers to insert and extract queue/port number */ -#define BRCM_TAG_SET_PORT_QUEUE(p, q) ((p) << 8 | q) -#define BRCM_TAG_GET_PORT(v) ((v) >> 8) -#define BRCM_TAG_GET_QUEUE(v) ((v) & 0xff) - - netdev_tx_t dsa_enqueue_skb(struct sk_buff *skb, struct net_device *dev); int dsa_port_get_phy_strings(struct dsa_port *dp, uint8_t *data); int dsa_port_get_ethtool_phy_stats(struct dsa_port *dp, uint64_t *data); diff --git a/net/dsa/tag_brcm.c b/net/dsa/tag_brcm.c index e934dace3922..e2577a7dcbca 100644 --- a/net/dsa/tag_brcm.c +++ b/net/dsa/tag_brcm.c @@ -5,6 +5,7 @@ * Copyright (C) 2014 Broadcom Corporation */ +#include #include #include #include -- cgit v1.2.3 From 424f481f06dc7f1528447cc3224f5fd3c5e11f65 Mon Sep 17 00:00:00 2001 From: Jonathan Lemon Date: Wed, 6 Jan 2021 14:18:29 -0800 Subject: skbuff: remove unused skb_zcopy_abort function skb_zcopy_abort() has no in-tree consumers, remove it. Signed-off-by: Jonathan Lemon Acked-by: Willem de Bruijn Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 333bcdc39635..3ca8d7c7b30c 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1490,17 +1490,6 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy) } } -/* Abort a zerocopy operation and revert zckey on error in send syscall */ -static inline void skb_zcopy_abort(struct sk_buff *skb) -{ - struct ubuf_info *uarg = skb_zcopy(skb); - - if (uarg) { - sock_zerocopy_put_abort(uarg, false); - skb_shinfo(skb)->tx_flags &= ~SKBTX_ZEROCOPY_FRAG; - } -} - static inline void skb_mark_not_on_list(struct sk_buff *skb) { skb->next = NULL; -- cgit v1.2.3 From 75518851a2a0c047def521aaf87db401de098352 Mon Sep 17 00:00:00 2001 From: Jonathan Lemon Date: Wed, 6 Jan 2021 14:18:31 -0800 Subject: skbuff: Push status and refcounts into sock_zerocopy_callback Before this change, the caller of sock_zerocopy_callback would need to save the zerocopy status, decrement and check the refcount, and then call the callback function - the callback was only invoked when the refcount reached zero. Now, the caller just passes the status into the callback function, which saves the status and handles its own refcounts. This makes the behavior of the sock_zerocopy_callback identical to the tpacket and vhost callbacks. Signed-off-by: Jonathan Lemon Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 3 --- net/core/skbuff.c | 14 +++++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 3ca8d7c7b30c..52e96c35f5af 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1479,9 +1479,6 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy) if (uarg) { if (skb_zcopy_is_nouarg(skb)) { /* no notification callback */ - } else if (uarg->callback == sock_zerocopy_callback) { - uarg->zerocopy = uarg->zerocopy && zerocopy; - sock_zerocopy_put(uarg); } else { uarg->callback(uarg, zerocopy); } diff --git a/net/core/skbuff.c b/net/core/skbuff.c index d88963f47f7d..8c18940723ff 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1194,7 +1194,7 @@ static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) return true; } -void sock_zerocopy_callback(struct ubuf_info *uarg, bool success) +static void __sock_zerocopy_callback(struct ubuf_info *uarg) { struct sk_buff *tail, *skb = skb_from_uarg(uarg); struct sock_exterr_skb *serr; @@ -1222,7 +1222,7 @@ void sock_zerocopy_callback(struct ubuf_info *uarg, bool success) serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY; serr->ee.ee_data = hi; serr->ee.ee_info = lo; - if (!success) + if (!uarg->zerocopy) serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED; q = &sk->sk_error_queue; @@ -1241,11 +1241,19 @@ release: consume_skb(skb); sock_put(sk); } + +void sock_zerocopy_callback(struct ubuf_info *uarg, bool success) +{ + uarg->zerocopy = uarg->zerocopy & success; + + if (refcount_dec_and_test(&uarg->refcnt)) + __sock_zerocopy_callback(uarg); +} EXPORT_SYMBOL_GPL(sock_zerocopy_callback); void sock_zerocopy_put(struct ubuf_info *uarg) { - if (uarg && refcount_dec_and_test(&uarg->refcnt)) + if (uarg) uarg->callback(uarg, uarg->zerocopy); } EXPORT_SYMBOL_GPL(sock_zerocopy_put); -- cgit v1.2.3 From 59776362b14b81dc45c6bbd4e7cb3871f390fc1b Mon Sep 17 00:00:00 2001 From: Jonathan Lemon Date: Wed, 6 Jan 2021 14:18:32 -0800 Subject: skbuff: replace sock_zerocopy_put() with skb_zcopy_put() Replace sock_zerocopy_put with the generic skb_zcopy_put() function. Pass 'true' as the success argument, as this is identical to no change. Signed-off-by: Jonathan Lemon Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 7 ++++++- net/core/skbuff.c | 9 +-------- net/ipv4/tcp.c | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 52e96c35f5af..a6c86839035b 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -496,7 +496,6 @@ static inline void sock_zerocopy_get(struct ubuf_info *uarg) refcount_inc(&uarg->refcnt); } -void sock_zerocopy_put(struct ubuf_info *uarg); void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref); void sock_zerocopy_callback(struct ubuf_info *uarg, bool success); @@ -1471,6 +1470,12 @@ static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb) return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL); } +static inline void skb_zcopy_put(struct ubuf_info *uarg) +{ + if (uarg) + uarg->callback(uarg, true); +} + /* Release a reference on a zerocopy structure */ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy) { diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 8c18940723ff..0e028825367a 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1251,13 +1251,6 @@ void sock_zerocopy_callback(struct ubuf_info *uarg, bool success) } EXPORT_SYMBOL_GPL(sock_zerocopy_callback); -void sock_zerocopy_put(struct ubuf_info *uarg) -{ - if (uarg) - uarg->callback(uarg, uarg->zerocopy); -} -EXPORT_SYMBOL_GPL(sock_zerocopy_put); - void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) { if (uarg) { @@ -1267,7 +1260,7 @@ void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) uarg->len--; if (have_uref) - sock_zerocopy_put(uarg); + skb_zcopy_put(uarg); } } EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index ed42d2193c5c..298a1fae841c 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1429,7 +1429,7 @@ out: tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); } out_nopush: - sock_zerocopy_put(uarg); + skb_zcopy_put(uarg); return copied + copied_syn; do_error: -- cgit v1.2.3 From e76d46cfff8d2335f363f58bd7387de4059d871d Mon Sep 17 00:00:00 2001 From: Jonathan Lemon Date: Wed, 6 Jan 2021 14:18:33 -0800 Subject: skbuff: replace sock_zerocopy_get with skb_zcopy_get Rename the get routines for consistency. Signed-off-by: Jonathan Lemon Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 12 ++++++------ net/core/skbuff.c | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index a6c86839035b..5b8a53ab51fd 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -491,11 +491,6 @@ struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size); struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size, struct ubuf_info *uarg); -static inline void sock_zerocopy_get(struct ubuf_info *uarg) -{ - refcount_inc(&uarg->refcnt); -} - void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref); void sock_zerocopy_callback(struct ubuf_info *uarg, bool success); @@ -1441,6 +1436,11 @@ static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb) return is_zcopy ? skb_uarg(skb) : NULL; } +static inline void skb_zcopy_get(struct ubuf_info *uarg) +{ + refcount_inc(&uarg->refcnt); +} + static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg, bool *have_ref) { @@ -1448,7 +1448,7 @@ static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg, if (unlikely(have_ref && *have_ref)) *have_ref = false; else - sock_zerocopy_get(uarg); + skb_zcopy_get(uarg); skb_shinfo(skb)->destructor_arg = uarg; skb_shinfo(skb)->tx_flags |= SKBTX_ZEROCOPY_FRAG; } diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 0e028825367a..00f195908e79 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1163,7 +1163,7 @@ struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size, /* no extra ref when appending to datagram (MSG_MORE) */ if (sk->sk_type == SOCK_STREAM) - sock_zerocopy_get(uarg); + skb_zcopy_get(uarg); return uarg; } -- cgit v1.2.3 From 36177832f42d9c7b222ab039678398a9d4070fff Mon Sep 17 00:00:00 2001 From: Jonathan Lemon Date: Wed, 6 Jan 2021 14:18:34 -0800 Subject: skbuff: Add skb parameter to the ubuf zerocopy callback Add an optional skb parameter to the zerocopy callback parameter, which is passed down from skb_zcopy_clear(). This gives access to the original skb, which is needed for upcoming RX zero-copy error handling. Signed-off-by: Jonathan Lemon Signed-off-by: Jakub Kicinski --- drivers/net/tap.c | 2 +- drivers/net/tun.c | 2 +- drivers/net/xen-netback/common.h | 3 ++- drivers/net/xen-netback/netback.c | 5 +++-- drivers/vhost/net.c | 3 ++- include/linux/skbuff.h | 17 ++++++++--------- net/core/skbuff.c | 3 ++- 7 files changed, 19 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/tap.c b/drivers/net/tap.c index 1f4bdd94407a..3f51f3766d18 100644 --- a/drivers/net/tap.c +++ b/drivers/net/tap.c @@ -727,7 +727,7 @@ static ssize_t tap_get_user(struct tap_queue *q, void *msg_control, skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; } else if (msg_control) { struct ubuf_info *uarg = msg_control; - uarg->callback(uarg, false); + uarg->callback(NULL, uarg, false); } if (tap) { diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 978ac0981d16..862c9063aa09 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1819,7 +1819,7 @@ drop: skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; } else if (msg_control) { struct ubuf_info *uarg = msg_control; - uarg->callback(uarg, false); + uarg->callback(NULL, uarg, false); } skb_reset_network_header(skb); diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h index 8ee24e351bdc..4a16d6e33c09 100644 --- a/drivers/net/xen-netback/common.h +++ b/drivers/net/xen-netback/common.h @@ -399,7 +399,8 @@ void xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb); void xenvif_carrier_on(struct xenvif *vif); /* Callback from stack when TX packet can be released */ -void xenvif_zerocopy_callback(struct ubuf_info *ubuf, bool zerocopy_success); +void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf, + bool zerocopy_success); /* Unmap a pending page and release it back to the guest */ void xenvif_idx_unmap(struct xenvif_queue *queue, u16 pending_idx); diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index bc3421d14576..19a27dce79d2 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c @@ -1091,7 +1091,7 @@ static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *s uarg = skb_shinfo(skb)->destructor_arg; /* increase inflight counter to offset decrement in callback */ atomic_inc(&queue->inflight_packets); - uarg->callback(uarg, true); + uarg->callback(NULL, uarg, true); skb_shinfo(skb)->destructor_arg = NULL; /* Fill the skb with the new (local) frags. */ @@ -1228,7 +1228,8 @@ static int xenvif_tx_submit(struct xenvif_queue *queue) return work_done; } -void xenvif_zerocopy_callback(struct ubuf_info *ubuf, bool zerocopy_success) +void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf, + bool zerocopy_success) { unsigned long flags; pending_ring_idx_t index; diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index c8784dfafdd7..f834a097895e 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -381,7 +381,8 @@ static void vhost_zerocopy_signal_used(struct vhost_net *net, } } -static void vhost_zerocopy_callback(struct ubuf_info *ubuf, bool success) +static void vhost_zerocopy_callback(struct sk_buff *skb, + struct ubuf_info *ubuf, bool success) { struct vhost_net_ubuf_ref *ubufs = ubuf->ctx; struct vhost_virtqueue *vq = ubufs->vq; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 5b8a53ab51fd..b23c3b4b3209 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -461,7 +461,8 @@ enum { * The desc field is used to track userspace buffer index. */ struct ubuf_info { - void (*callback)(struct ubuf_info *, bool zerocopy_success); + void (*callback)(struct sk_buff *, struct ubuf_info *, + bool zerocopy_success); union { struct { unsigned long desc; @@ -493,7 +494,8 @@ struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size, void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref); -void sock_zerocopy_callback(struct ubuf_info *uarg, bool success); +void sock_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, + bool success); int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len); int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, @@ -1473,20 +1475,17 @@ static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb) static inline void skb_zcopy_put(struct ubuf_info *uarg) { if (uarg) - uarg->callback(uarg, true); + uarg->callback(NULL, uarg, true); } /* Release a reference on a zerocopy structure */ -static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy) +static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success) { struct ubuf_info *uarg = skb_zcopy(skb); if (uarg) { - if (skb_zcopy_is_nouarg(skb)) { - /* no notification callback */ - } else { - uarg->callback(uarg, zerocopy); - } + if (!skb_zcopy_is_nouarg(skb)) + uarg->callback(skb, uarg, zerocopy_success); skb_shinfo(skb)->tx_flags &= ~SKBTX_ZEROCOPY_FRAG; } diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 00f195908e79..89130b21d9f0 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1242,7 +1242,8 @@ release: sock_put(sk); } -void sock_zerocopy_callback(struct ubuf_info *uarg, bool success) +void sock_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, + bool success) { uarg->zerocopy = uarg->zerocopy & success; -- cgit v1.2.3 From 236a6b1cd585a408139550201343f3f16f9324b9 Mon Sep 17 00:00:00 2001 From: Jonathan Lemon Date: Wed, 6 Jan 2021 14:18:35 -0800 Subject: skbuff: Call sock_zerocopy_put_abort from skb_zcopy_put_abort The sock_zerocopy_put_abort function contains logic which is specific to the current zerocopy implementation. Add a wrapper which checks the callback and dispatches apppropriately. Signed-off-by: Jonathan Lemon Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 10 ++++++++++ net/core/skbuff.c | 12 +++++------- net/ipv4/ip_output.c | 3 +-- net/ipv4/tcp.c | 2 +- net/ipv6/ip6_output.c | 3 +-- 5 files changed, 18 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index b23c3b4b3209..9f7393167f0a 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1478,6 +1478,16 @@ static inline void skb_zcopy_put(struct ubuf_info *uarg) uarg->callback(NULL, uarg, true); } +static inline void skb_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref) +{ + if (uarg) { + if (uarg->callback == sock_zerocopy_callback) + sock_zerocopy_put_abort(uarg, have_uref); + else if (have_uref) + skb_zcopy_put(uarg); + } +} + /* Release a reference on a zerocopy structure */ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success) { diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 89130b21d9f0..5b9cd528d6a6 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1254,15 +1254,13 @@ EXPORT_SYMBOL_GPL(sock_zerocopy_callback); void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) { - if (uarg) { - struct sock *sk = skb_from_uarg(uarg)->sk; + struct sock *sk = skb_from_uarg(uarg)->sk; - atomic_dec(&sk->sk_zckey); - uarg->len--; + atomic_dec(&sk->sk_zckey); + uarg->len--; - if (have_uref) - skb_zcopy_put(uarg); - } + if (have_uref) + sock_zerocopy_callback(NULL, uarg, true); } EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort); diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 89fff5f59eea..bae9b29e17a3 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -1230,8 +1230,7 @@ alloc_new_skb: error_efault: err = -EFAULT; error: - if (uarg) - sock_zerocopy_put_abort(uarg, extra_uref); + skb_zcopy_put_abort(uarg, extra_uref); cork->length -= length; IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS); refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 298a1fae841c..fb58215972ba 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1440,7 +1440,7 @@ do_fault: if (copied + copied_syn) goto out; out_err: - sock_zerocopy_put_abort(uarg, true); + skb_zcopy_put_abort(uarg, true); err = sk_stream_error(sk, flags, err); /* make sure we wake any epoll edge trigger waiter */ if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index 749ad72386b2..c8c87891533a 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -1715,8 +1715,7 @@ alloc_new_skb: error_efault: err = -EFAULT; error: - if (uarg) - sock_zerocopy_put_abort(uarg, extra_uref); + skb_zcopy_put_abort(uarg, extra_uref); cork->length -= length; IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); -- cgit v1.2.3 From 8c793822c5803e01d03f71c431f59316f0b278b7 Mon Sep 17 00:00:00 2001 From: Jonathan Lemon Date: Wed, 6 Jan 2021 14:18:37 -0800 Subject: skbuff: rename sock_zerocopy_* to msg_zerocopy_* At Willem's suggestion, rename the sock_zerocopy_* functions so that they match the MSG_ZEROCOPY flag, which makes it clear they are specific to this zerocopy implementation. Signed-off-by: Jonathan Lemon Acked-by: Willem de Bruijn Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 18 +++++++++--------- net/core/skbuff.c | 30 +++++++++++++++--------------- net/ipv4/ip_output.c | 2 +- net/ipv4/tcp.c | 2 +- net/ipv6/ip6_output.c | 2 +- 5 files changed, 27 insertions(+), 27 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 9f7393167f0a..f6a3cee5c967 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -488,14 +488,14 @@ struct ubuf_info { int mm_account_pinned_pages(struct mmpin *mmp, size_t size); void mm_unaccount_pinned_pages(struct mmpin *mmp); -struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size); -struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size, - struct ubuf_info *uarg); +struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size); +struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, + struct ubuf_info *uarg); -void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref); +void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref); -void sock_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, - bool success); +void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, + bool success); int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len); int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, @@ -1481,8 +1481,8 @@ static inline void skb_zcopy_put(struct ubuf_info *uarg) static inline void skb_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref) { if (uarg) { - if (uarg->callback == sock_zerocopy_callback) - sock_zerocopy_put_abort(uarg, have_uref); + if (uarg->callback == msg_zerocopy_callback) + msg_zerocopy_put_abort(uarg, have_uref); else if (have_uref) skb_zcopy_put(uarg); } @@ -2776,7 +2776,7 @@ static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask) if (likely(!skb_zcopy(skb))) return 0; if (!skb_zcopy_is_nouarg(skb) && - skb_uarg(skb)->callback == sock_zerocopy_callback) + skb_uarg(skb)->callback == msg_zerocopy_callback) return 0; return skb_copy_ubufs(skb, gfp_mask); } diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 6d031ed99182..bcd56763952e 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1094,7 +1094,7 @@ void mm_unaccount_pinned_pages(struct mmpin *mmp) } EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages); -struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size) +struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) { struct ubuf_info *uarg; struct sk_buff *skb; @@ -1114,7 +1114,7 @@ struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size) return NULL; } - uarg->callback = sock_zerocopy_callback; + uarg->callback = msg_zerocopy_callback; uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; uarg->len = 1; uarg->bytelen = size; @@ -1124,15 +1124,15 @@ struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size) return uarg; } -EXPORT_SYMBOL_GPL(sock_zerocopy_alloc); +EXPORT_SYMBOL_GPL(msg_zerocopy_alloc); static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg) { return container_of((void *)uarg, struct sk_buff, cb); } -struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size, - struct ubuf_info *uarg) +struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, + struct ubuf_info *uarg) { if (uarg) { const u32 byte_limit = 1 << 19; /* limit to a few TSO */ @@ -1171,9 +1171,9 @@ struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size, } new_alloc: - return sock_zerocopy_alloc(sk, size); + return msg_zerocopy_alloc(sk, size); } -EXPORT_SYMBOL_GPL(sock_zerocopy_realloc); +EXPORT_SYMBOL_GPL(msg_zerocopy_realloc); static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) { @@ -1195,7 +1195,7 @@ static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) return true; } -static void __sock_zerocopy_callback(struct ubuf_info *uarg) +static void __msg_zerocopy_callback(struct ubuf_info *uarg) { struct sk_buff *tail, *skb = skb_from_uarg(uarg); struct sock_exterr_skb *serr; @@ -1243,17 +1243,17 @@ release: sock_put(sk); } -void sock_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, - bool success) +void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, + bool success) { uarg->zerocopy = uarg->zerocopy & success; if (refcount_dec_and_test(&uarg->refcnt)) - __sock_zerocopy_callback(uarg); + __msg_zerocopy_callback(uarg); } -EXPORT_SYMBOL_GPL(sock_zerocopy_callback); +EXPORT_SYMBOL_GPL(msg_zerocopy_callback); -void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) +void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) { struct sock *sk = skb_from_uarg(uarg)->sk; @@ -1261,9 +1261,9 @@ void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) uarg->len--; if (have_uref) - sock_zerocopy_callback(NULL, uarg, true); + msg_zerocopy_callback(NULL, uarg, true); } -EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort); +EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort); int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len) { diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index bae9b29e17a3..ffee03729285 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -1018,7 +1018,7 @@ static int __ip_append_data(struct sock *sk, csummode = CHECKSUM_PARTIAL; if (flags & MSG_ZEROCOPY && length && sock_flag(sk, SOCK_ZEROCOPY)) { - uarg = sock_zerocopy_realloc(sk, length, skb_zcopy(skb)); + uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb)); if (!uarg) return -ENOBUFS; extra_uref = !skb_zcopy(skb); /* only ref on new uarg */ diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index fb58215972ba..2882d520f5b1 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1217,7 +1217,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) if (flags & MSG_ZEROCOPY && size && sock_flag(sk, SOCK_ZEROCOPY)) { skb = tcp_write_queue_tail(sk); - uarg = sock_zerocopy_realloc(sk, size, skb_zcopy(skb)); + uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb)); if (!uarg) { err = -ENOBUFS; goto out_err; diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index c8c87891533a..f59cfa39686a 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -1471,7 +1471,7 @@ emsgsize: csummode = CHECKSUM_PARTIAL; if (flags & MSG_ZEROCOPY && length && sock_flag(sk, SOCK_ZEROCOPY)) { - uarg = sock_zerocopy_realloc(sk, length, skb_zcopy(skb)); + uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb)); if (!uarg) return -ENOBUFS; extra_uref = !skb_zcopy(skb); /* only ref on new uarg */ -- cgit v1.2.3 From 06b4feb37e64e543714c971a4162a75e2e4024d4 Mon Sep 17 00:00:00 2001 From: Jonathan Lemon Date: Wed, 6 Jan 2021 14:18:38 -0800 Subject: net: group skb_shinfo zerocopy related bits together. In preparation for expanded zerocopy (TX and RX), move the zerocopy related bits out of tx_flags into their own flag word. Signed-off-by: Jonathan Lemon Signed-off-by: Jakub Kicinski --- drivers/net/tap.c | 3 +-- drivers/net/tun.c | 3 +-- drivers/net/xen-netback/interface.c | 4 ++-- include/linux/skbuff.h | 38 ++++++++++++++++++++----------------- net/core/skbuff.c | 9 ++++----- net/ipv4/tcp.c | 2 +- net/kcm/kcmsock.c | 4 ++-- 7 files changed, 32 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/tap.c b/drivers/net/tap.c index 3f51f3766d18..f7a19d9b7c27 100644 --- a/drivers/net/tap.c +++ b/drivers/net/tap.c @@ -723,8 +723,7 @@ static ssize_t tap_get_user(struct tap_queue *q, void *msg_control, /* copy skb_ubuf_info for callback when skb has no error */ if (zerocopy) { skb_shinfo(skb)->destructor_arg = msg_control; - skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY; - skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; + skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; } else if (msg_control) { struct ubuf_info *uarg = msg_control; uarg->callback(NULL, uarg, false); diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 862c9063aa09..d8f1c3d34612 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1815,8 +1815,7 @@ drop: /* copy skb_ubuf_info for callback when skb has no error */ if (zerocopy) { skb_shinfo(skb)->destructor_arg = msg_control; - skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY; - skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; + skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; } else if (msg_control) { struct ubuf_info *uarg = msg_control; uarg->callback(NULL, uarg, false); diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c index acb786d8b1d8..08b0e3d0b7eb 100644 --- a/drivers/net/xen-netback/interface.c +++ b/drivers/net/xen-netback/interface.c @@ -47,7 +47,7 @@ /* Number of bytes allowed on the internal guest Rx queue. */ #define XENVIF_RX_QUEUE_BYTES (XEN_NETIF_RX_RING_SIZE/2 * PAGE_SIZE) -/* This function is used to set SKBTX_DEV_ZEROCOPY as well as +/* This function is used to set SKBFL_ZEROCOPY_ENABLE as well as * increasing the inflight counter. We need to increase the inflight * counter because core driver calls into xenvif_zerocopy_callback * which calls xenvif_skb_zerocopy_complete. @@ -55,7 +55,7 @@ void xenvif_skb_zerocopy_prepare(struct xenvif_queue *queue, struct sk_buff *skb) { - skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY; + skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_ENABLE; atomic_inc(&queue->inflight_packets); } diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index f6a3cee5c967..7b5adc511ac6 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -430,28 +430,32 @@ enum { /* device driver is going to provide hardware time stamp */ SKBTX_IN_PROGRESS = 1 << 2, - /* device driver supports TX zero-copy buffers */ - SKBTX_DEV_ZEROCOPY = 1 << 3, - /* generate wifi status information (where possible) */ SKBTX_WIFI_STATUS = 1 << 4, - /* This indicates at least one fragment might be overwritten - * (as in vmsplice(), sendfile() ...) - * If we need to compute a TX checksum, we'll need to copy - * all frags to avoid possible bad checksum - */ - SKBTX_SHARED_FRAG = 1 << 5, - /* generate software time stamp when entering packet scheduling */ SKBTX_SCHED_TSTAMP = 1 << 6, }; -#define SKBTX_ZEROCOPY_FRAG (SKBTX_DEV_ZEROCOPY | SKBTX_SHARED_FRAG) #define SKBTX_ANY_SW_TSTAMP (SKBTX_SW_TSTAMP | \ SKBTX_SCHED_TSTAMP) #define SKBTX_ANY_TSTAMP (SKBTX_HW_TSTAMP | SKBTX_ANY_SW_TSTAMP) +/* Definitions for flags in struct skb_shared_info */ +enum { + /* use zcopy routines */ + SKBFL_ZEROCOPY_ENABLE = BIT(0), + + /* This indicates at least one fragment might be overwritten + * (as in vmsplice(), sendfile() ...) + * If we need to compute a TX checksum, we'll need to copy + * all frags to avoid possible bad checksum + */ + SKBFL_SHARED_FRAG = BIT(1), +}; + +#define SKBFL_ZEROCOPY_FRAG (SKBFL_ZEROCOPY_ENABLE | SKBFL_SHARED_FRAG) + /* * The callback notifies userspace to release buffers when skb DMA is done in * lower device, the skb last reference should be 0 when calling this. @@ -506,7 +510,7 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, * the end of the header data, ie. at skb->end. */ struct skb_shared_info { - __u8 __unused; + __u8 flags; __u8 meta_len; __u8 nr_frags; __u8 tx_flags; @@ -1433,7 +1437,7 @@ static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb) static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb) { - bool is_zcopy = skb && skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY; + bool is_zcopy = skb && skb_shinfo(skb)->flags & SKBFL_ZEROCOPY_ENABLE; return is_zcopy ? skb_uarg(skb) : NULL; } @@ -1452,14 +1456,14 @@ static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg, else skb_zcopy_get(uarg); skb_shinfo(skb)->destructor_arg = uarg; - skb_shinfo(skb)->tx_flags |= SKBTX_ZEROCOPY_FRAG; + skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; } } static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val) { skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL); - skb_shinfo(skb)->tx_flags |= SKBTX_ZEROCOPY_FRAG; + skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; } static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb) @@ -1497,7 +1501,7 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success) if (!skb_zcopy_is_nouarg(skb)) uarg->callback(skb, uarg, zerocopy_success); - skb_shinfo(skb)->tx_flags &= ~SKBTX_ZEROCOPY_FRAG; + skb_shinfo(skb)->flags &= ~SKBFL_ZEROCOPY_FRAG; } } @@ -3323,7 +3327,7 @@ static inline int skb_linearize(struct sk_buff *skb) static inline bool skb_has_shared_frag(const struct sk_buff *skb) { return skb_is_nonlinear(skb) && - skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG; + skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; } /** diff --git a/net/core/skbuff.c b/net/core/skbuff.c index bcd56763952e..5b9e52cbd087 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1327,7 +1327,7 @@ static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, * @skb: the skb to modify * @gfp_mask: allocation priority * - * This must be called on SKBTX_DEV_ZEROCOPY skb. + * This must be called on skb with SKBFL_ZEROCOPY_ENABLE. * It will copy all frags into kernel and drop the reference * to userspace pages. * @@ -3264,8 +3264,7 @@ void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) { int pos = skb_headlen(skb); - skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags & - SKBTX_SHARED_FRAG; + skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; skb_zerocopy_clone(skb1, skb, 0); if (len < pos) /* Split line is inside header. */ skb_split_inside_header(skb, skb1, len, pos); @@ -3954,8 +3953,8 @@ normal: skb_copy_from_linear_data_offset(head_skb, offset, skb_put(nskb, hsize), hsize); - skb_shinfo(nskb)->tx_flags |= skb_shinfo(head_skb)->tx_flags & - SKBTX_SHARED_FRAG; + skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags & + SKBFL_SHARED_FRAG; if (skb_orphan_frags(frag_skb, GFP_ATOMIC) || skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 2882d520f5b1..1954190b33c7 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1010,7 +1010,7 @@ new_segment: } if (!(flags & MSG_NO_SHARED_FRAGS)) - skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; + skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG; skb->len += copy; skb->data_len += copy; diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index a9eb616f5521..d0b56ffbb057 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -786,7 +786,7 @@ static ssize_t kcm_sendpage(struct socket *sock, struct page *page, if (skb_can_coalesce(skb, i, page, offset)) { skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); - skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; + skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG; goto coalesced; } @@ -834,7 +834,7 @@ static ssize_t kcm_sendpage(struct socket *sock, struct page *page, get_page(page); skb_fill_page_desc(skb, i, page, offset, size); - skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; + skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG; coalesced: skb->len += size; -- cgit v1.2.3 From 04c2d33eabdc76df730e1e356f6b2c656381d7d5 Mon Sep 17 00:00:00 2001 From: Jonathan Lemon Date: Wed, 6 Jan 2021 14:18:39 -0800 Subject: skbuff: add flags to ubuf_info for ubuf setup Currently, when an ubuf is attached to a new skb, the shared flags word is initialized to a fixed value. Instead of doing this, set the default flags in the ubuf, and have new skbs inherit from this default. This is needed when setting up different zerocopy types. Signed-off-by: Jonathan Lemon Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 3 ++- net/core/skbuff.c | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 7b5adc511ac6..a7bc71d2debb 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -480,6 +480,7 @@ struct ubuf_info { }; }; refcount_t refcnt; + u8 flags; struct mmpin { struct user_struct *user; @@ -1456,7 +1457,7 @@ static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg, else skb_zcopy_get(uarg); skb_shinfo(skb)->destructor_arg = uarg; - skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; + skb_shinfo(skb)->flags |= uarg->flags; } } diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 5b9e52cbd087..a7bcbbff99e3 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1119,6 +1119,7 @@ struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) uarg->len = 1; uarg->bytelen = size; uarg->zerocopy = 1; + uarg->flags = SKBFL_ZEROCOPY_FRAG; refcount_set(&uarg->refcnt, 1); sock_hold(sk); -- cgit v1.2.3 From 9ee5e5ade033875191a2d2e470033e9cdde44a6a Mon Sep 17 00:00:00 2001 From: Jonathan Lemon Date: Wed, 6 Jan 2021 14:18:40 -0800 Subject: tap/tun: add skb_zcopy_init() helper for initialization. Replace direct assignments with skb_zcopy_init() for zerocopy cases where a new skb is initialized, without changing the reference counts. Signed-off-by: Jonathan Lemon Signed-off-by: Jakub Kicinski --- drivers/net/tap.c | 3 +-- drivers/net/tun.c | 3 +-- drivers/vhost/net.c | 1 + include/linux/skbuff.h | 9 +++++++-- 4 files changed, 10 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/tap.c b/drivers/net/tap.c index f7a19d9b7c27..3c652c8ac5ba 100644 --- a/drivers/net/tap.c +++ b/drivers/net/tap.c @@ -722,8 +722,7 @@ static ssize_t tap_get_user(struct tap_queue *q, void *msg_control, tap = rcu_dereference(q->tap); /* copy skb_ubuf_info for callback when skb has no error */ if (zerocopy) { - skb_shinfo(skb)->destructor_arg = msg_control; - skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; + skb_zcopy_init(skb, msg_control); } else if (msg_control) { struct ubuf_info *uarg = msg_control; uarg->callback(NULL, uarg, false); diff --git a/drivers/net/tun.c b/drivers/net/tun.c index d8f1c3d34612..02a93cfdb6b1 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1814,8 +1814,7 @@ drop: /* copy skb_ubuf_info for callback when skb has no error */ if (zerocopy) { - skb_shinfo(skb)->destructor_arg = msg_control; - skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; + skb_zcopy_init(skb, msg_control); } else if (msg_control) { struct ubuf_info *uarg = msg_control; uarg->callback(NULL, uarg, false); diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index f834a097895e..3b744031ec8f 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -903,6 +903,7 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock) ubuf->callback = vhost_zerocopy_callback; ubuf->ctx = nvq->ubufs; ubuf->desc = nvq->upend_idx; + ubuf->flags = SKBFL_ZEROCOPY_FRAG; refcount_set(&ubuf->refcnt, 1); msg.msg_control = &ctl; ctl.type = TUN_MSG_UBUF; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index a7bc71d2debb..68ad45a98810 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1448,6 +1448,12 @@ static inline void skb_zcopy_get(struct ubuf_info *uarg) refcount_inc(&uarg->refcnt); } +static inline void skb_zcopy_init(struct sk_buff *skb, struct ubuf_info *uarg) +{ + skb_shinfo(skb)->destructor_arg = uarg; + skb_shinfo(skb)->flags |= uarg->flags; +} + static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg, bool *have_ref) { @@ -1456,8 +1462,7 @@ static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg, *have_ref = false; else skb_zcopy_get(uarg); - skb_shinfo(skb)->destructor_arg = uarg; - skb_shinfo(skb)->flags |= uarg->flags; + skb_zcopy_init(skb, uarg); } } -- cgit v1.2.3 From 8e0449172497a915e79da66b222255dc9b4a5f31 Mon Sep 17 00:00:00 2001 From: Jonathan Lemon Date: Wed, 6 Jan 2021 14:18:41 -0800 Subject: skbuff: Rename skb_zcopy_{get|put} to net_zcopy_{get|put} Unlike the rest of the skb_zcopy_ functions, these routines operate on a 'struct ubuf', not a skb. Remove the 'skb_' prefix from the naming to make things clearer. Suggested-by: Willem de Bruijn Signed-off-by: Jonathan Lemon Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 10 +++++----- net/core/skbuff.c | 2 +- net/ipv4/ip_output.c | 2 +- net/ipv4/tcp.c | 4 ++-- net/ipv6/ip6_output.c | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 68ad45a98810..7a057b1f1eb8 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1443,7 +1443,7 @@ static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb) return is_zcopy ? skb_uarg(skb) : NULL; } -static inline void skb_zcopy_get(struct ubuf_info *uarg) +static inline void net_zcopy_get(struct ubuf_info *uarg) { refcount_inc(&uarg->refcnt); } @@ -1461,7 +1461,7 @@ static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg, if (unlikely(have_ref && *have_ref)) *have_ref = false; else - skb_zcopy_get(uarg); + net_zcopy_get(uarg); skb_zcopy_init(skb, uarg); } } @@ -1482,19 +1482,19 @@ static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb) return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL); } -static inline void skb_zcopy_put(struct ubuf_info *uarg) +static inline void net_zcopy_put(struct ubuf_info *uarg) { if (uarg) uarg->callback(NULL, uarg, true); } -static inline void skb_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref) +static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref) { if (uarg) { if (uarg->callback == msg_zerocopy_callback) msg_zerocopy_put_abort(uarg, have_uref); else if (have_uref) - skb_zcopy_put(uarg); + net_zcopy_put(uarg); } } diff --git a/net/core/skbuff.c b/net/core/skbuff.c index a7bcbbff99e3..7626a33cce59 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1165,7 +1165,7 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, /* no extra ref when appending to datagram (MSG_MORE) */ if (sk->sk_type == SOCK_STREAM) - skb_zcopy_get(uarg); + net_zcopy_get(uarg); return uarg; } diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index ffee03729285..102b1998ba3c 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -1230,7 +1230,7 @@ alloc_new_skb: error_efault: err = -EFAULT; error: - skb_zcopy_put_abort(uarg, extra_uref); + net_zcopy_put_abort(uarg, extra_uref); cork->length -= length; IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS); refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 1954190b33c7..2267d21c73a6 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1429,7 +1429,7 @@ out: tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); } out_nopush: - skb_zcopy_put(uarg); + net_zcopy_put(uarg); return copied + copied_syn; do_error: @@ -1440,7 +1440,7 @@ do_fault: if (copied + copied_syn) goto out; out_err: - skb_zcopy_put_abort(uarg, true); + net_zcopy_put_abort(uarg, true); err = sk_stream_error(sk, flags, err); /* make sure we wake any epoll edge trigger waiter */ if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index f59cfa39686a..072ce9678616 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -1715,7 +1715,7 @@ alloc_new_skb: error_efault: err = -EFAULT; error: - skb_zcopy_put_abort(uarg, extra_uref); + net_zcopy_put_abort(uarg, extra_uref); cork->length -= length; IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc); -- cgit v1.2.3 From 994122211665301080cd232c06ae219b681dcb57 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Thu, 7 Jan 2021 17:34:01 -0600 Subject: remoteproc: qcom: expose types for COMPILE_TEST Stub functions are defined for SSR notifier registration in case QCOM_RPROC_COMMON is not configured. As a result, code that uses these functions can link successfully even if the common remoteproc code is not built. Code that registers an SSR notifier function likely needs the types defined in "qcom_rproc.h", but those are only exposed if QCOM_RPROC_COMMON is enabled. Rearrange the conditional definition so the qcom_ssr_notify_data structure and qcom_ssr_notify_type enumerated type are defined whether or not QCOM_RPROC_COMMON is enabled. Reviewed-by: Bjorn Andersson Signed-off-by: Alex Elder Signed-off-by: Jakub Kicinski --- include/linux/remoteproc/qcom_rproc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/remoteproc/qcom_rproc.h b/include/linux/remoteproc/qcom_rproc.h index 647051662174..82b211518136 100644 --- a/include/linux/remoteproc/qcom_rproc.h +++ b/include/linux/remoteproc/qcom_rproc.h @@ -3,8 +3,6 @@ struct notifier_block; -#if IS_ENABLED(CONFIG_QCOM_RPROC_COMMON) - /** * enum qcom_ssr_notify_type - Startup/Shutdown events related to a remoteproc * processor. @@ -26,6 +24,8 @@ struct qcom_ssr_notify_data { bool crashed; }; +#if IS_ENABLED(CONFIG_QCOM_RPROC_COMMON) + void *qcom_register_ssr_notifier(const char *name, struct notifier_block *nb); int qcom_unregister_ssr_notifier(void *notify, struct notifier_block *nb); -- cgit v1.2.3 From ce2ceb9b1cff777c7d8beb368eddc3b8e45381f6 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Thu, 7 Jan 2021 17:34:02 -0600 Subject: soc: qcom: mdt_loader: define stubs for COMPILE_TEST Define stub functions for the exposed MDT functions in case QCOM_MDT_LOADER is not configured. This allows users of these functions to link correctly for COMPILE_TEST builds without QCOM_SCM enabled. Reviewed-by: Bjorn Andersson Signed-off-by: Alex Elder Signed-off-by: Jakub Kicinski --- include/linux/soc/qcom/mdt_loader.h | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) (limited to 'include/linux') diff --git a/include/linux/soc/qcom/mdt_loader.h b/include/linux/soc/qcom/mdt_loader.h index e600baec6825..afd47217996b 100644 --- a/include/linux/soc/qcom/mdt_loader.h +++ b/include/linux/soc/qcom/mdt_loader.h @@ -11,6 +11,8 @@ struct device; struct firmware; +#if IS_ENABLED(CONFIG_QCOM_MDT_LOADER) + ssize_t qcom_mdt_get_size(const struct firmware *fw); int qcom_mdt_load(struct device *dev, const struct firmware *fw, const char *fw_name, int pas_id, void *mem_region, @@ -23,4 +25,37 @@ int qcom_mdt_load_no_init(struct device *dev, const struct firmware *fw, phys_addr_t *reloc_base); void *qcom_mdt_read_metadata(const struct firmware *fw, size_t *data_len); +#else /* !IS_ENABLED(CONFIG_QCOM_MDT_LOADER) */ + +static inline ssize_t qcom_mdt_get_size(const struct firmware *fw) +{ + return -ENODEV; +} + +static inline int qcom_mdt_load(struct device *dev, const struct firmware *fw, + const char *fw_name, int pas_id, + void *mem_region, phys_addr_t mem_phys, + size_t mem_size, phys_addr_t *reloc_base) +{ + return -ENODEV; +} + +static inline int qcom_mdt_load_no_init(struct device *dev, + const struct firmware *fw, + const char *fw_name, int pas_id, + void *mem_region, phys_addr_t mem_phys, + size_t mem_size, + phys_addr_t *reloc_base) +{ + return -ENODEV; +} + +static inline void *qcom_mdt_read_metadata(const struct firmware *fw, + size_t *data_len) +{ + return ERR_PTR(-ENODEV); +} + +#endif /* !IS_ENABLED(CONFIG_QCOM_MDT_LOADER) */ + #endif -- cgit v1.2.3 From 1d11fa696733ffb9ac24771716b1b1b9953e5a48 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Fri, 8 Jan 2021 03:39:03 -0800 Subject: net-gro: remove GRO_DROP GRO_DROP can only be returned from napi_gro_frags() if the skb has not been allocated by a prior napi_get_frags() Since drivers must use napi_get_frags() and test its result before populating the skb with metadata, we can safely remove GRO_DROP since it offers no practical use. Signed-off-by: Eric Dumazet Cc: Jesse Brandeburg Acked-by: Edward Cree Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 1 - net/core/dev.c | 11 ----------- 2 files changed, 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 1ec3ac5d5bbf..5b949076ed23 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -376,7 +376,6 @@ enum gro_result { GRO_MERGED_FREE, GRO_HELD, GRO_NORMAL, - GRO_DROP, GRO_CONSUMED, }; typedef enum gro_result gro_result_t; diff --git a/net/core/dev.c b/net/core/dev.c index 7afbb642e203..e4d77c8abe76 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -6070,10 +6070,6 @@ static gro_result_t napi_skb_finish(struct napi_struct *napi, gro_normal_one(napi, skb); break; - case GRO_DROP: - kfree_skb(skb); - break; - case GRO_MERGED_FREE: if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) napi_skb_free_stolen_head(skb); @@ -6158,10 +6154,6 @@ static gro_result_t napi_frags_finish(struct napi_struct *napi, gro_normal_one(napi, skb); break; - case GRO_DROP: - napi_reuse_skb(napi, skb); - break; - case GRO_MERGED_FREE: if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) napi_skb_free_stolen_head(skb); @@ -6223,9 +6215,6 @@ gro_result_t napi_gro_frags(struct napi_struct *napi) gro_result_t ret; struct sk_buff *skb = napi_frags_skb(napi); - if (!skb) - return GRO_DROP; - trace_napi_gro_frags_entry(skb); ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); -- cgit v1.2.3 From a643bff752dcf72a07e1b2ab2f8587e4f51118be Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 11 Jan 2021 23:55:14 -0800 Subject: bpf: Add bpf_patch_call_args prototype to include/linux/bpf.h Add bpf_patch_call_args() prototype. This function is called from BPF verifier and only if CONFIG_BPF_JIT_ALWAYS_ON is not defined. This fixes compiler warning about missing prototype in some kernel configurations. Fixes: 1ea47e01ad6e ("bpf: add support for bpf_call to interpreter") Reported-by: kernel test robot Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20210112075520.4103414-2-andrii@kernel.org --- include/linux/bpf.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 07cb5d15e743..ef9309604b3e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1403,7 +1403,10 @@ static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) /* verify correctness of eBPF program */ int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, union bpf_attr __user *uattr); + +#ifndef CONFIG_BPF_JIT_ALWAYS_ON void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); +#endif struct btf *bpf_get_btf_vmlinux(void); -- cgit v1.2.3 From 6943c2b05bf09fd5c5729f7d7d803bf3f126cb9a Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 11 Jan 2021 23:55:15 -0800 Subject: bpf: Avoid warning when re-casting __bpf_call_base into __bpf_call_base_args BPF interpreter uses extra input argument, so re-casts __bpf_call_base into __bpf_call_base_args. Avoid compiler warning about incompatible function prototypes by casting to void * first. Fixes: 1ea47e01ad6e ("bpf: add support for bpf_call to interpreter") Reported-by: kernel test robot Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20210112075520.4103414-3-andrii@kernel.org --- include/linux/filter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/filter.h b/include/linux/filter.h index 29c27656165b..5edf2b660881 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -886,7 +886,7 @@ void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp); u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); #define __bpf_call_base_args \ ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \ - __bpf_call_base) + (void *)__bpf_call_base) struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog); void bpf_jit_compile(struct bpf_prog *prog); -- cgit v1.2.3 From 936f8946bdb48239f4292812d4d2e26c6d328c95 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 11 Jan 2021 23:55:16 -0800 Subject: bpf: Declare __bpf_free_used_maps() unconditionally __bpf_free_used_maps() is always defined in kernel/bpf/core.c, while include/linux/bpf.h is guarding it behind CONFIG_BPF_SYSCALL. Move it out of that guard region and fix compiler warning. Fixes: a2ea07465c8d ("bpf: Fix missing prog untrack in release_maps") Reported-by: kernel test robot Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20210112075520.4103414-4-andrii@kernel.org --- include/linux/bpf.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index ef9309604b3e..6e585dbc10df 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1206,8 +1206,6 @@ void bpf_prog_sub(struct bpf_prog *prog, int i); void bpf_prog_inc(struct bpf_prog *prog); struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); void bpf_prog_put(struct bpf_prog *prog); -void __bpf_free_used_maps(struct bpf_prog_aux *aux, - struct bpf_map **used_maps, u32 len); void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock); void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock); @@ -1676,6 +1674,9 @@ static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, return bpf_prog_get_type_dev(ufd, type, false); } +void __bpf_free_used_maps(struct bpf_prog_aux *aux, + struct bpf_map **used_maps, u32 len); + bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); int bpf_prog_offload_compile(struct bpf_prog *prog); -- cgit v1.2.3 From 541c3bad8dc51b253ba8686d0cd7628e6b9b5f4c Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Mon, 11 Jan 2021 23:55:18 -0800 Subject: bpf: Support BPF ksym variables in kernel modules Add support for directly accessing kernel module variables from BPF programs using special ldimm64 instructions. This functionality builds upon vmlinux ksym support, but extends ldimm64 with src_reg=BPF_PSEUDO_BTF_ID to allow specifying kernel module BTF's FD in insn[1].imm field. During BPF program load time, verifier will resolve FD to BTF object and will take reference on BTF object itself and, for module BTFs, corresponding module as well, to make sure it won't be unloaded from under running BPF program. The mechanism used is similar to how bpf_prog keeps track of used bpf_maps. One interesting change is also in how per-CPU variable is determined. The logic is to find .data..percpu data section in provided BTF, but both vmlinux and module each have their own .data..percpu entries in BTF. So for module's case, the search for DATASEC record needs to look at only module's added BTF types. This is implemented with custom search function. Signed-off-by: Andrii Nakryiko Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Acked-by: Hao Luo Link: https://lore.kernel.org/bpf/20210112075520.4103414-6-andrii@kernel.org --- include/linux/bpf.h | 10 +++ include/linux/bpf_verifier.h | 3 + include/linux/btf.h | 3 + kernel/bpf/btf.c | 31 ++++++++- kernel/bpf/core.c | 23 +++++++ kernel/bpf/verifier.c | 154 +++++++++++++++++++++++++++++++++++-------- 6 files changed, 194 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 6e585dbc10df..1aac2af12fed 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -761,9 +761,15 @@ struct bpf_ctx_arg_aux { u32 btf_id; }; +struct btf_mod_pair { + struct btf *btf; + struct module *module; +}; + struct bpf_prog_aux { atomic64_t refcnt; u32 used_map_cnt; + u32 used_btf_cnt; u32 max_ctx_offset; u32 max_pkt_offset; u32 max_tp_access; @@ -802,6 +808,7 @@ struct bpf_prog_aux { const struct bpf_prog_ops *ops; struct bpf_map **used_maps; struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */ + struct btf_mod_pair *used_btfs; struct bpf_prog *prog; struct user_struct *user; u64 load_time; /* ns since boottime */ @@ -1668,6 +1675,9 @@ bpf_base_func_proto(enum bpf_func_id func_id) } #endif /* CONFIG_BPF_SYSCALL */ +void __bpf_free_used_btfs(struct bpf_prog_aux *aux, + struct btf_mod_pair *used_btfs, u32 len); + static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type) { diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index e941fe1484e5..dfe6f85d97dd 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -340,6 +340,7 @@ struct bpf_insn_aux_data { }; #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ +#define MAX_USED_BTFS 64 /* max number of BTFs accessed by one BPF program */ #define BPF_VERIFIER_TMP_LOG_SIZE 1024 @@ -398,7 +399,9 @@ struct bpf_verifier_env { struct bpf_verifier_state_list **explored_states; /* search pruning optimization */ struct bpf_verifier_state_list *free_list; struct bpf_map *used_maps[MAX_USED_MAPS]; /* array of map's used by eBPF program */ + struct btf_mod_pair used_btfs[MAX_USED_BTFS]; /* array of BTF's used by BPF program */ u32 used_map_cnt; /* number of used maps */ + u32 used_btf_cnt; /* number of used BTF objects */ u32 id_gen; /* used to generate unique reg IDs */ bool allow_ptr_leaks; bool allow_ptr_to_map_access; diff --git a/include/linux/btf.h b/include/linux/btf.h index 4c200f5d242b..7fabf1428093 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -91,6 +91,9 @@ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, int btf_get_fd_by_id(u32 id); u32 btf_obj_id(const struct btf *btf); bool btf_is_kernel(const struct btf *btf); +bool btf_is_module(const struct btf *btf); +struct module *btf_try_get_module(const struct btf *btf); +u32 btf_nr_types(const struct btf *btf); bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, const struct btf_member *m, u32 expected_offset, u32 expected_size); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 8d6bdb4f4d61..7ccc0133723a 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -458,7 +458,7 @@ static bool btf_type_is_datasec(const struct btf_type *t) return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC; } -static u32 btf_nr_types_total(const struct btf *btf) +u32 btf_nr_types(const struct btf *btf) { u32 total = 0; @@ -476,7 +476,7 @@ s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind) const char *tname; u32 i, total; - total = btf_nr_types_total(btf); + total = btf_nr_types(btf); for (i = 1; i < total; i++) { t = btf_type_by_id(btf, i); if (BTF_INFO_KIND(t->info) != kind) @@ -5743,6 +5743,11 @@ bool btf_is_kernel(const struct btf *btf) return btf->kernel_btf; } +bool btf_is_module(const struct btf *btf) +{ + return btf->kernel_btf && strcmp(btf->name, "vmlinux") != 0; +} + static int btf_id_cmp_func(const void *a, const void *b) { const int *pa = a, *pb = b; @@ -5877,3 +5882,25 @@ static int __init btf_module_init(void) fs_initcall(btf_module_init); #endif /* CONFIG_DEBUG_INFO_BTF_MODULES */ + +struct module *btf_try_get_module(const struct btf *btf) +{ + struct module *res = NULL; +#ifdef CONFIG_DEBUG_INFO_BTF_MODULES + struct btf_module *btf_mod, *tmp; + + mutex_lock(&btf_module_mutex); + list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { + if (btf_mod->btf != btf) + continue; + + if (try_module_get(btf_mod->module)) + res = btf_mod->module; + + break; + } + mutex_unlock(&btf_module_mutex); +#endif + + return res; +} diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 261f8692d0d2..69c3c308de5e 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2119,6 +2119,28 @@ static void bpf_free_used_maps(struct bpf_prog_aux *aux) kfree(aux->used_maps); } +void __bpf_free_used_btfs(struct bpf_prog_aux *aux, + struct btf_mod_pair *used_btfs, u32 len) +{ +#ifdef CONFIG_BPF_SYSCALL + struct btf_mod_pair *btf_mod; + u32 i; + + for (i = 0; i < len; i++) { + btf_mod = &used_btfs[i]; + if (btf_mod->module) + module_put(btf_mod->module); + btf_put(btf_mod->btf); + } +#endif +} + +static void bpf_free_used_btfs(struct bpf_prog_aux *aux) +{ + __bpf_free_used_btfs(aux, aux->used_btfs, aux->used_btf_cnt); + kfree(aux->used_btfs); +} + static void bpf_prog_free_deferred(struct work_struct *work) { struct bpf_prog_aux *aux; @@ -2126,6 +2148,7 @@ static void bpf_prog_free_deferred(struct work_struct *work) aux = container_of(work, struct bpf_prog_aux, work); bpf_free_used_maps(aux); + bpf_free_used_btfs(aux); if (bpf_prog_is_dev_bound(aux)) bpf_prog_offload_destroy(aux->prog); #ifdef CONFIG_PERF_EVENTS diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5534e667bdb1..ae2aee48cf82 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9703,6 +9703,36 @@ process_bpf_exit: return 0; } +static int find_btf_percpu_datasec(struct btf *btf) +{ + const struct btf_type *t; + const char *tname; + int i, n; + + /* + * Both vmlinux and module each have their own ".data..percpu" + * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF + * types to look at only module's own BTF types. + */ + n = btf_nr_types(btf); + if (btf_is_module(btf)) + i = btf_nr_types(btf_vmlinux); + else + i = 1; + + for(; i < n; i++) { + t = btf_type_by_id(btf, i); + if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) + continue; + + tname = btf_name_by_offset(btf, t->name_off); + if (!strcmp(tname, ".data..percpu")) + return i; + } + + return -ENOENT; +} + /* replace pseudo btf_id with kernel symbol address */ static int check_pseudo_btf_id(struct bpf_verifier_env *env, struct bpf_insn *insn, @@ -9710,48 +9740,57 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, { const struct btf_var_secinfo *vsi; const struct btf_type *datasec; + struct btf_mod_pair *btf_mod; const struct btf_type *t; const char *sym_name; bool percpu = false; u32 type, id = insn->imm; + struct btf *btf; s32 datasec_id; u64 addr; - int i; - - if (!btf_vmlinux) { - verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); - return -EINVAL; - } + int i, btf_fd, err; - if (insn[1].imm != 0) { - verbose(env, "reserved field (insn[1].imm) is used in pseudo_btf_id ldimm64 insn.\n"); - return -EINVAL; + btf_fd = insn[1].imm; + if (btf_fd) { + btf = btf_get_by_fd(btf_fd); + if (IS_ERR(btf)) { + verbose(env, "invalid module BTF object FD specified.\n"); + return -EINVAL; + } + } else { + if (!btf_vmlinux) { + verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); + return -EINVAL; + } + btf = btf_vmlinux; + btf_get(btf); } - t = btf_type_by_id(btf_vmlinux, id); + t = btf_type_by_id(btf, id); if (!t) { verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); - return -ENOENT; + err = -ENOENT; + goto err_put; } if (!btf_type_is_var(t)) { - verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", - id); - return -EINVAL; + verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id); + err = -EINVAL; + goto err_put; } - sym_name = btf_name_by_offset(btf_vmlinux, t->name_off); + sym_name = btf_name_by_offset(btf, t->name_off); addr = kallsyms_lookup_name(sym_name); if (!addr) { verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", sym_name); - return -ENOENT; + err = -ENOENT; + goto err_put; } - datasec_id = btf_find_by_name_kind(btf_vmlinux, ".data..percpu", - BTF_KIND_DATASEC); + datasec_id = find_btf_percpu_datasec(btf); if (datasec_id > 0) { - datasec = btf_type_by_id(btf_vmlinux, datasec_id); + datasec = btf_type_by_id(btf, datasec_id); for_each_vsi(i, datasec, vsi) { if (vsi->type == id) { percpu = true; @@ -9764,10 +9803,10 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, insn[1].imm = addr >> 32; type = t->type; - t = btf_type_skip_modifiers(btf_vmlinux, type, NULL); + t = btf_type_skip_modifiers(btf, type, NULL); if (percpu) { aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID; - aux->btf_var.btf = btf_vmlinux; + aux->btf_var.btf = btf; aux->btf_var.btf_id = type; } else if (!btf_type_is_struct(t)) { const struct btf_type *ret; @@ -9775,21 +9814,54 @@ static int check_pseudo_btf_id(struct bpf_verifier_env *env, u32 tsize; /* resolve the type size of ksym. */ - ret = btf_resolve_size(btf_vmlinux, t, &tsize); + ret = btf_resolve_size(btf, t, &tsize); if (IS_ERR(ret)) { - tname = btf_name_by_offset(btf_vmlinux, t->name_off); + tname = btf_name_by_offset(btf, t->name_off); verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", tname, PTR_ERR(ret)); - return -EINVAL; + err = -EINVAL; + goto err_put; } aux->btf_var.reg_type = PTR_TO_MEM; aux->btf_var.mem_size = tsize; } else { aux->btf_var.reg_type = PTR_TO_BTF_ID; - aux->btf_var.btf = btf_vmlinux; + aux->btf_var.btf = btf; aux->btf_var.btf_id = type; } + + /* check whether we recorded this BTF (and maybe module) already */ + for (i = 0; i < env->used_btf_cnt; i++) { + if (env->used_btfs[i].btf == btf) { + btf_put(btf); + return 0; + } + } + + if (env->used_btf_cnt >= MAX_USED_BTFS) { + err = -E2BIG; + goto err_put; + } + + btf_mod = &env->used_btfs[env->used_btf_cnt]; + btf_mod->btf = btf; + btf_mod->module = NULL; + + /* if we reference variables from kernel module, bump its refcount */ + if (btf_is_module(btf)) { + btf_mod->module = btf_try_get_module(btf); + if (!btf_mod->module) { + err = -ENXIO; + goto err_put; + } + } + + env->used_btf_cnt++; + return 0; +err_put: + btf_put(btf); + return err; } static int check_map_prealloc(struct bpf_map *map) @@ -10086,6 +10158,13 @@ static void release_maps(struct bpf_verifier_env *env) env->used_map_cnt); } +/* drop refcnt of maps used by the rejected program */ +static void release_btfs(struct bpf_verifier_env *env) +{ + __bpf_free_used_btfs(env->prog->aux, env->used_btfs, + env->used_btf_cnt); +} + /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) { @@ -12098,7 +12177,10 @@ skip_full_check: goto err_release_maps; } - if (ret == 0 && env->used_map_cnt) { + if (ret) + goto err_release_maps; + + if (env->used_map_cnt) { /* if program passed verifier, update used_maps in bpf_prog_info */ env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, sizeof(env->used_maps[0]), @@ -12112,15 +12194,29 @@ skip_full_check: memcpy(env->prog->aux->used_maps, env->used_maps, sizeof(env->used_maps[0]) * env->used_map_cnt); env->prog->aux->used_map_cnt = env->used_map_cnt; + } + if (env->used_btf_cnt) { + /* if program passed verifier, update used_btfs in bpf_prog_aux */ + env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, + sizeof(env->used_btfs[0]), + GFP_KERNEL); + if (!env->prog->aux->used_btfs) { + ret = -ENOMEM; + goto err_release_maps; + } + memcpy(env->prog->aux->used_btfs, env->used_btfs, + sizeof(env->used_btfs[0]) * env->used_btf_cnt); + env->prog->aux->used_btf_cnt = env->used_btf_cnt; + } + if (env->used_map_cnt || env->used_btf_cnt) { /* program is valid. Convert pseudo bpf_ld_imm64 into generic * bpf_ld_imm64 instructions */ convert_pseudo_ld_imm64(env); } - if (ret == 0) - adjust_btf_func(env); + adjust_btf_func(env); err_release_maps: if (!env->prog->aux->used_maps) @@ -12128,6 +12224,8 @@ err_release_maps: * them now. Otherwise free_used_maps() will release them. */ release_maps(env); + if (!env->prog->aux->used_btfs) + release_btfs(env); /* extension progs temporarily inherit the attach_type of their targets for verification purposes, so set it back to zero before returning -- cgit v1.2.3 From 5a9d5ecd69ed65fc2d49cdecfc1c1ab1e4d7af34 Mon Sep 17 00:00:00 2001 From: Marc Kleine-Budde Date: Mon, 11 Jan 2021 15:19:18 +0100 Subject: can: dev: move bittiming related code into seperate file This patch moves the bittiming related code of the CAN device infrastructure into a separate file. Reviewed-by: Vincent Mailhol Link: https://lore.kernel.org/r/20210111141930.693847-4-mkl@pengutronix.de Signed-off-by: Marc Kleine-Budde --- MAINTAINERS | 1 + drivers/net/can/dev/Makefile | 1 + drivers/net/can/dev/bittiming.c | 261 ++++++++++++++++++++++++++++++++++++++++ drivers/net/can/dev/dev.c | 261 ---------------------------------------- include/linux/can/bittiming.h | 44 +++++++ include/linux/can/dev.h | 16 +-- 6 files changed, 308 insertions(+), 276 deletions(-) create mode 100644 drivers/net/can/dev/bittiming.c create mode 100644 include/linux/can/bittiming.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index c3091a91ebbf..d17662df1cd7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3943,6 +3943,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/mkl/linux-can.git T: git git://git.kernel.org/pub/scm/linux/kernel/git/mkl/linux-can-next.git F: Documentation/devicetree/bindings/net/can/ F: drivers/net/can/ +F: include/linux/can/bittiming.h F: include/linux/can/dev.h F: include/linux/can/led.h F: include/linux/can/platform/ diff --git a/drivers/net/can/dev/Makefile b/drivers/net/can/dev/Makefile index cba92e6bcf6f..b5c6bb848d9d 100644 --- a/drivers/net/can/dev/Makefile +++ b/drivers/net/can/dev/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CAN_DEV) += can-dev.o +can-dev-y += bittiming.o can-dev-y += dev.o can-dev-y += rx-offload.o diff --git a/drivers/net/can/dev/bittiming.c b/drivers/net/can/dev/bittiming.c new file mode 100644 index 000000000000..f7fe226bb395 --- /dev/null +++ b/drivers/net/can/dev/bittiming.c @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2005 Marc Kleine-Budde, Pengutronix + * Copyright (C) 2006 Andrey Volkov, Varma Electronics + * Copyright (C) 2008-2009 Wolfgang Grandegger + */ + +#include + +#ifdef CONFIG_CAN_CALC_BITTIMING +#define CAN_CALC_MAX_ERROR 50 /* in one-tenth of a percent */ + +/* Bit-timing calculation derived from: + * + * Code based on LinCAN sources and H8S2638 project + * Copyright 2004-2006 Pavel Pisa - DCE FELK CVUT cz + * Copyright 2005 Stanislav Marek + * email: pisa@cmp.felk.cvut.cz + * + * Calculates proper bit-timing parameters for a specified bit-rate + * and sample-point, which can then be used to set the bit-timing + * registers of the CAN controller. You can find more information + * in the header file linux/can/netlink.h. + */ +static int +can_update_sample_point(const struct can_bittiming_const *btc, + unsigned int sample_point_nominal, unsigned int tseg, + unsigned int *tseg1_ptr, unsigned int *tseg2_ptr, + unsigned int *sample_point_error_ptr) +{ + unsigned int sample_point_error, best_sample_point_error = UINT_MAX; + unsigned int sample_point, best_sample_point = 0; + unsigned int tseg1, tseg2; + int i; + + for (i = 0; i <= 1; i++) { + tseg2 = tseg + CAN_SYNC_SEG - + (sample_point_nominal * (tseg + CAN_SYNC_SEG)) / + 1000 - i; + tseg2 = clamp(tseg2, btc->tseg2_min, btc->tseg2_max); + tseg1 = tseg - tseg2; + if (tseg1 > btc->tseg1_max) { + tseg1 = btc->tseg1_max; + tseg2 = tseg - tseg1; + } + + sample_point = 1000 * (tseg + CAN_SYNC_SEG - tseg2) / + (tseg + CAN_SYNC_SEG); + sample_point_error = abs(sample_point_nominal - sample_point); + + if (sample_point <= sample_point_nominal && + sample_point_error < best_sample_point_error) { + best_sample_point = sample_point; + best_sample_point_error = sample_point_error; + *tseg1_ptr = tseg1; + *tseg2_ptr = tseg2; + } + } + + if (sample_point_error_ptr) + *sample_point_error_ptr = best_sample_point_error; + + return best_sample_point; +} + +int can_calc_bittiming(struct net_device *dev, struct can_bittiming *bt, + const struct can_bittiming_const *btc) +{ + struct can_priv *priv = netdev_priv(dev); + unsigned int bitrate; /* current bitrate */ + unsigned int bitrate_error; /* difference between current and nominal value */ + unsigned int best_bitrate_error = UINT_MAX; + unsigned int sample_point_error; /* difference between current and nominal value */ + unsigned int best_sample_point_error = UINT_MAX; + unsigned int sample_point_nominal; /* nominal sample point */ + unsigned int best_tseg = 0; /* current best value for tseg */ + unsigned int best_brp = 0; /* current best value for brp */ + unsigned int brp, tsegall, tseg, tseg1 = 0, tseg2 = 0; + u64 v64; + + /* Use CiA recommended sample points */ + if (bt->sample_point) { + sample_point_nominal = bt->sample_point; + } else { + if (bt->bitrate > 800000) + sample_point_nominal = 750; + else if (bt->bitrate > 500000) + sample_point_nominal = 800; + else + sample_point_nominal = 875; + } + + /* tseg even = round down, odd = round up */ + for (tseg = (btc->tseg1_max + btc->tseg2_max) * 2 + 1; + tseg >= (btc->tseg1_min + btc->tseg2_min) * 2; tseg--) { + tsegall = CAN_SYNC_SEG + tseg / 2; + + /* Compute all possible tseg choices (tseg=tseg1+tseg2) */ + brp = priv->clock.freq / (tsegall * bt->bitrate) + tseg % 2; + + /* choose brp step which is possible in system */ + brp = (brp / btc->brp_inc) * btc->brp_inc; + if (brp < btc->brp_min || brp > btc->brp_max) + continue; + + bitrate = priv->clock.freq / (brp * tsegall); + bitrate_error = abs(bt->bitrate - bitrate); + + /* tseg brp biterror */ + if (bitrate_error > best_bitrate_error) + continue; + + /* reset sample point error if we have a better bitrate */ + if (bitrate_error < best_bitrate_error) + best_sample_point_error = UINT_MAX; + + can_update_sample_point(btc, sample_point_nominal, tseg / 2, + &tseg1, &tseg2, &sample_point_error); + if (sample_point_error > best_sample_point_error) + continue; + + best_sample_point_error = sample_point_error; + best_bitrate_error = bitrate_error; + best_tseg = tseg / 2; + best_brp = brp; + + if (bitrate_error == 0 && sample_point_error == 0) + break; + } + + if (best_bitrate_error) { + /* Error in one-tenth of a percent */ + v64 = (u64)best_bitrate_error * 1000; + do_div(v64, bt->bitrate); + bitrate_error = (u32)v64; + if (bitrate_error > CAN_CALC_MAX_ERROR) { + netdev_err(dev, + "bitrate error %d.%d%% too high\n", + bitrate_error / 10, bitrate_error % 10); + return -EDOM; + } + netdev_warn(dev, "bitrate error %d.%d%%\n", + bitrate_error / 10, bitrate_error % 10); + } + + /* real sample point */ + bt->sample_point = can_update_sample_point(btc, sample_point_nominal, + best_tseg, &tseg1, &tseg2, + NULL); + + v64 = (u64)best_brp * 1000 * 1000 * 1000; + do_div(v64, priv->clock.freq); + bt->tq = (u32)v64; + bt->prop_seg = tseg1 / 2; + bt->phase_seg1 = tseg1 - bt->prop_seg; + bt->phase_seg2 = tseg2; + + /* check for sjw user settings */ + if (!bt->sjw || !btc->sjw_max) { + bt->sjw = 1; + } else { + /* bt->sjw is at least 1 -> sanitize upper bound to sjw_max */ + if (bt->sjw > btc->sjw_max) + bt->sjw = btc->sjw_max; + /* bt->sjw must not be higher than tseg2 */ + if (tseg2 < bt->sjw) + bt->sjw = tseg2; + } + + bt->brp = best_brp; + + /* real bitrate */ + bt->bitrate = priv->clock.freq / + (bt->brp * (CAN_SYNC_SEG + tseg1 + tseg2)); + + return 0; +} +#endif /* CONFIG_CAN_CALC_BITTIMING */ + +/* Checks the validity of the specified bit-timing parameters prop_seg, + * phase_seg1, phase_seg2 and sjw and tries to determine the bitrate + * prescaler value brp. You can find more information in the header + * file linux/can/netlink.h. + */ +static int can_fixup_bittiming(struct net_device *dev, struct can_bittiming *bt, + const struct can_bittiming_const *btc) +{ + struct can_priv *priv = netdev_priv(dev); + int tseg1, alltseg; + u64 brp64; + + tseg1 = bt->prop_seg + bt->phase_seg1; + if (!bt->sjw) + bt->sjw = 1; + if (bt->sjw > btc->sjw_max || + tseg1 < btc->tseg1_min || tseg1 > btc->tseg1_max || + bt->phase_seg2 < btc->tseg2_min || bt->phase_seg2 > btc->tseg2_max) + return -ERANGE; + + brp64 = (u64)priv->clock.freq * (u64)bt->tq; + if (btc->brp_inc > 1) + do_div(brp64, btc->brp_inc); + brp64 += 500000000UL - 1; + do_div(brp64, 1000000000UL); /* the practicable BRP */ + if (btc->brp_inc > 1) + brp64 *= btc->brp_inc; + bt->brp = (u32)brp64; + + if (bt->brp < btc->brp_min || bt->brp > btc->brp_max) + return -EINVAL; + + alltseg = bt->prop_seg + bt->phase_seg1 + bt->phase_seg2 + 1; + bt->bitrate = priv->clock.freq / (bt->brp * alltseg); + bt->sample_point = ((tseg1 + 1) * 1000) / alltseg; + + return 0; +} + +/* Checks the validity of predefined bitrate settings */ +static int +can_validate_bitrate(struct net_device *dev, struct can_bittiming *bt, + const u32 *bitrate_const, + const unsigned int bitrate_const_cnt) +{ + struct can_priv *priv = netdev_priv(dev); + unsigned int i; + + for (i = 0; i < bitrate_const_cnt; i++) { + if (bt->bitrate == bitrate_const[i]) + break; + } + + if (i >= priv->bitrate_const_cnt) + return -EINVAL; + + return 0; +} + +int can_get_bittiming(struct net_device *dev, struct can_bittiming *bt, + const struct can_bittiming_const *btc, + const u32 *bitrate_const, + const unsigned int bitrate_const_cnt) +{ + int err; + + /* Depending on the given can_bittiming parameter structure the CAN + * timing parameters are calculated based on the provided bitrate OR + * alternatively the CAN timing parameters (tq, prop_seg, etc.) are + * provided directly which are then checked and fixed up. + */ + if (!bt->tq && bt->bitrate && btc) + err = can_calc_bittiming(dev, bt, btc); + else if (bt->tq && !bt->bitrate && btc) + err = can_fixup_bittiming(dev, bt, btc); + else if (!bt->tq && bt->bitrate && bitrate_const) + err = can_validate_bitrate(dev, bt, bitrate_const, + bitrate_const_cnt); + else + err = -EINVAL; + + return err; +} diff --git a/drivers/net/can/dev/dev.c b/drivers/net/can/dev/dev.c index 3486704c8a95..1b3ab95b3fd1 100644 --- a/drivers/net/can/dev/dev.c +++ b/drivers/net/can/dev/dev.c @@ -58,267 +58,6 @@ u8 can_fd_len2dlc(u8 len) } EXPORT_SYMBOL_GPL(can_fd_len2dlc); -#ifdef CONFIG_CAN_CALC_BITTIMING -#define CAN_CALC_MAX_ERROR 50 /* in one-tenth of a percent */ - -/* Bit-timing calculation derived from: - * - * Code based on LinCAN sources and H8S2638 project - * Copyright 2004-2006 Pavel Pisa - DCE FELK CVUT cz - * Copyright 2005 Stanislav Marek - * email: pisa@cmp.felk.cvut.cz - * - * Calculates proper bit-timing parameters for a specified bit-rate - * and sample-point, which can then be used to set the bit-timing - * registers of the CAN controller. You can find more information - * in the header file linux/can/netlink.h. - */ -static int -can_update_sample_point(const struct can_bittiming_const *btc, - unsigned int sample_point_nominal, unsigned int tseg, - unsigned int *tseg1_ptr, unsigned int *tseg2_ptr, - unsigned int *sample_point_error_ptr) -{ - unsigned int sample_point_error, best_sample_point_error = UINT_MAX; - unsigned int sample_point, best_sample_point = 0; - unsigned int tseg1, tseg2; - int i; - - for (i = 0; i <= 1; i++) { - tseg2 = tseg + CAN_SYNC_SEG - - (sample_point_nominal * (tseg + CAN_SYNC_SEG)) / - 1000 - i; - tseg2 = clamp(tseg2, btc->tseg2_min, btc->tseg2_max); - tseg1 = tseg - tseg2; - if (tseg1 > btc->tseg1_max) { - tseg1 = btc->tseg1_max; - tseg2 = tseg - tseg1; - } - - sample_point = 1000 * (tseg + CAN_SYNC_SEG - tseg2) / - (tseg + CAN_SYNC_SEG); - sample_point_error = abs(sample_point_nominal - sample_point); - - if (sample_point <= sample_point_nominal && - sample_point_error < best_sample_point_error) { - best_sample_point = sample_point; - best_sample_point_error = sample_point_error; - *tseg1_ptr = tseg1; - *tseg2_ptr = tseg2; - } - } - - if (sample_point_error_ptr) - *sample_point_error_ptr = best_sample_point_error; - - return best_sample_point; -} - -static int can_calc_bittiming(struct net_device *dev, struct can_bittiming *bt, - const struct can_bittiming_const *btc) -{ - struct can_priv *priv = netdev_priv(dev); - unsigned int bitrate; /* current bitrate */ - unsigned int bitrate_error; /* difference between current and nominal value */ - unsigned int best_bitrate_error = UINT_MAX; - unsigned int sample_point_error; /* difference between current and nominal value */ - unsigned int best_sample_point_error = UINT_MAX; - unsigned int sample_point_nominal; /* nominal sample point */ - unsigned int best_tseg = 0; /* current best value for tseg */ - unsigned int best_brp = 0; /* current best value for brp */ - unsigned int brp, tsegall, tseg, tseg1 = 0, tseg2 = 0; - u64 v64; - - /* Use CiA recommended sample points */ - if (bt->sample_point) { - sample_point_nominal = bt->sample_point; - } else { - if (bt->bitrate > 800000) - sample_point_nominal = 750; - else if (bt->bitrate > 500000) - sample_point_nominal = 800; - else - sample_point_nominal = 875; - } - - /* tseg even = round down, odd = round up */ - for (tseg = (btc->tseg1_max + btc->tseg2_max) * 2 + 1; - tseg >= (btc->tseg1_min + btc->tseg2_min) * 2; tseg--) { - tsegall = CAN_SYNC_SEG + tseg / 2; - - /* Compute all possible tseg choices (tseg=tseg1+tseg2) */ - brp = priv->clock.freq / (tsegall * bt->bitrate) + tseg % 2; - - /* choose brp step which is possible in system */ - brp = (brp / btc->brp_inc) * btc->brp_inc; - if (brp < btc->brp_min || brp > btc->brp_max) - continue; - - bitrate = priv->clock.freq / (brp * tsegall); - bitrate_error = abs(bt->bitrate - bitrate); - - /* tseg brp biterror */ - if (bitrate_error > best_bitrate_error) - continue; - - /* reset sample point error if we have a better bitrate */ - if (bitrate_error < best_bitrate_error) - best_sample_point_error = UINT_MAX; - - can_update_sample_point(btc, sample_point_nominal, tseg / 2, - &tseg1, &tseg2, &sample_point_error); - if (sample_point_error > best_sample_point_error) - continue; - - best_sample_point_error = sample_point_error; - best_bitrate_error = bitrate_error; - best_tseg = tseg / 2; - best_brp = brp; - - if (bitrate_error == 0 && sample_point_error == 0) - break; - } - - if (best_bitrate_error) { - /* Error in one-tenth of a percent */ - v64 = (u64)best_bitrate_error * 1000; - do_div(v64, bt->bitrate); - bitrate_error = (u32)v64; - if (bitrate_error > CAN_CALC_MAX_ERROR) { - netdev_err(dev, - "bitrate error %d.%d%% too high\n", - bitrate_error / 10, bitrate_error % 10); - return -EDOM; - } - netdev_warn(dev, "bitrate error %d.%d%%\n", - bitrate_error / 10, bitrate_error % 10); - } - - /* real sample point */ - bt->sample_point = can_update_sample_point(btc, sample_point_nominal, - best_tseg, &tseg1, &tseg2, - NULL); - - v64 = (u64)best_brp * 1000 * 1000 * 1000; - do_div(v64, priv->clock.freq); - bt->tq = (u32)v64; - bt->prop_seg = tseg1 / 2; - bt->phase_seg1 = tseg1 - bt->prop_seg; - bt->phase_seg2 = tseg2; - - /* check for sjw user settings */ - if (!bt->sjw || !btc->sjw_max) { - bt->sjw = 1; - } else { - /* bt->sjw is at least 1 -> sanitize upper bound to sjw_max */ - if (bt->sjw > btc->sjw_max) - bt->sjw = btc->sjw_max; - /* bt->sjw must not be higher than tseg2 */ - if (tseg2 < bt->sjw) - bt->sjw = tseg2; - } - - bt->brp = best_brp; - - /* real bitrate */ - bt->bitrate = priv->clock.freq / - (bt->brp * (CAN_SYNC_SEG + tseg1 + tseg2)); - - return 0; -} -#else /* !CONFIG_CAN_CALC_BITTIMING */ -static int can_calc_bittiming(struct net_device *dev, struct can_bittiming *bt, - const struct can_bittiming_const *btc) -{ - netdev_err(dev, "bit-timing calculation not available\n"); - return -EINVAL; -} -#endif /* CONFIG_CAN_CALC_BITTIMING */ - -/* Checks the validity of the specified bit-timing parameters prop_seg, - * phase_seg1, phase_seg2 and sjw and tries to determine the bitrate - * prescaler value brp. You can find more information in the header - * file linux/can/netlink.h. - */ -static int can_fixup_bittiming(struct net_device *dev, struct can_bittiming *bt, - const struct can_bittiming_const *btc) -{ - struct can_priv *priv = netdev_priv(dev); - int tseg1, alltseg; - u64 brp64; - - tseg1 = bt->prop_seg + bt->phase_seg1; - if (!bt->sjw) - bt->sjw = 1; - if (bt->sjw > btc->sjw_max || - tseg1 < btc->tseg1_min || tseg1 > btc->tseg1_max || - bt->phase_seg2 < btc->tseg2_min || bt->phase_seg2 > btc->tseg2_max) - return -ERANGE; - - brp64 = (u64)priv->clock.freq * (u64)bt->tq; - if (btc->brp_inc > 1) - do_div(brp64, btc->brp_inc); - brp64 += 500000000UL - 1; - do_div(brp64, 1000000000UL); /* the practicable BRP */ - if (btc->brp_inc > 1) - brp64 *= btc->brp_inc; - bt->brp = (u32)brp64; - - if (bt->brp < btc->brp_min || bt->brp > btc->brp_max) - return -EINVAL; - - alltseg = bt->prop_seg + bt->phase_seg1 + bt->phase_seg2 + 1; - bt->bitrate = priv->clock.freq / (bt->brp * alltseg); - bt->sample_point = ((tseg1 + 1) * 1000) / alltseg; - - return 0; -} - -/* Checks the validity of predefined bitrate settings */ -static int -can_validate_bitrate(struct net_device *dev, struct can_bittiming *bt, - const u32 *bitrate_const, - const unsigned int bitrate_const_cnt) -{ - struct can_priv *priv = netdev_priv(dev); - unsigned int i; - - for (i = 0; i < bitrate_const_cnt; i++) { - if (bt->bitrate == bitrate_const[i]) - break; - } - - if (i >= priv->bitrate_const_cnt) - return -EINVAL; - - return 0; -} - -static int can_get_bittiming(struct net_device *dev, struct can_bittiming *bt, - const struct can_bittiming_const *btc, - const u32 *bitrate_const, - const unsigned int bitrate_const_cnt) -{ - int err; - - /* Depending on the given can_bittiming parameter structure the CAN - * timing parameters are calculated based on the provided bitrate OR - * alternatively the CAN timing parameters (tq, prop_seg, etc.) are - * provided directly which are then checked and fixed up. - */ - if (!bt->tq && bt->bitrate && btc) - err = can_calc_bittiming(dev, bt, btc); - else if (bt->tq && !bt->bitrate && btc) - err = can_fixup_bittiming(dev, bt, btc); - else if (!bt->tq && bt->bitrate && bitrate_const) - err = can_validate_bitrate(dev, bt, bitrate_const, - bitrate_const_cnt); - else - err = -EINVAL; - - return err; -} - static void can_update_state_error_stats(struct net_device *dev, enum can_state new_state) { diff --git a/include/linux/can/bittiming.h b/include/linux/can/bittiming.h new file mode 100644 index 000000000000..707575c668f4 --- /dev/null +++ b/include/linux/can/bittiming.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2020 Pengutronix, Marc Kleine-Budde + */ + +#ifndef _CAN_BITTIMING_H +#define _CAN_BITTIMING_H + +#include +#include + +#define CAN_SYNC_SEG 1 + +#ifdef CONFIG_CAN_CALC_BITTIMING +int can_calc_bittiming(struct net_device *dev, struct can_bittiming *bt, + const struct can_bittiming_const *btc); +#else /* !CONFIG_CAN_CALC_BITTIMING */ +static inline int +can_calc_bittiming(struct net_device *dev, struct can_bittiming *bt, + const struct can_bittiming_const *btc) +{ + netdev_err(dev, "bit-timing calculation not available\n"); + return -EINVAL; +} +#endif /* CONFIG_CAN_CALC_BITTIMING */ + +int can_get_bittiming(struct net_device *dev, struct can_bittiming *bt, + const struct can_bittiming_const *btc, + const u32 *bitrate_const, + const unsigned int bitrate_const_cnt); + +/* + * can_bit_time() - Duration of one bit + * + * Please refer to ISO 11898-1:2015, section 11.3.1.1 "Bit time" for + * additional information. + * + * Return: the number of time quanta in one bit. + */ +static inline unsigned int can_bit_time(const struct can_bittiming *bt) +{ + return CAN_SYNC_SEG + bt->prop_seg + bt->phase_seg1 + bt->phase_seg2; +} + +#endif /* !_CAN_BITTIMING_H */ diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h index 197a79535cc2..054c3bed190b 100644 --- a/include/linux/can/dev.h +++ b/include/linux/can/dev.h @@ -15,6 +15,7 @@ #define _CAN_DEV_H #include +#include #include #include #include @@ -82,21 +83,6 @@ struct can_priv { #endif }; -#define CAN_SYNC_SEG 1 - -/* - * can_bit_time() - Duration of one bit - * - * Please refer to ISO 11898-1:2015, section 11.3.1.1 "Bit time" for - * additional information. - * - * Return: the number of time quanta in one bit. - */ -static inline unsigned int can_bit_time(const struct can_bittiming *bt) -{ - return CAN_SYNC_SEG + bt->prop_seg + bt->phase_seg1 + bt->phase_seg2; -} - /* * can_cc_dlc2len(value) - convert a given data length code (dlc) of a * Classical CAN frame into a valid data length of max. 8 bytes. -- cgit v1.2.3 From bdd2e413192dd5f2153d166cd907b048cce872e8 Mon Sep 17 00:00:00 2001 From: Marc Kleine-Budde Date: Mon, 11 Jan 2021 15:19:19 +0100 Subject: can: dev: move length related code into seperate file This patch moves all CAN frame length related code of the CAN device infrastructure into a separate file. Reviewed-by: Vincent Mailhol Link: https://lore.kernel.org/r/20210111141930.693847-5-mkl@pengutronix.de Signed-off-by: Marc Kleine-Budde --- MAINTAINERS | 1 + drivers/net/can/dev/Makefile | 1 + drivers/net/can/dev/dev.c | 33 ------------------------------ drivers/net/can/dev/length.c | 38 +++++++++++++++++++++++++++++++++++ include/linux/can/dev.h | 41 +------------------------------------ include/linux/can/length.h | 48 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 89 insertions(+), 73 deletions(-) create mode 100644 drivers/net/can/dev/length.c create mode 100644 include/linux/can/length.h (limited to 'include/linux') diff --git a/MAINTAINERS b/MAINTAINERS index d17662df1cd7..21780c4e2e71 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3946,6 +3946,7 @@ F: drivers/net/can/ F: include/linux/can/bittiming.h F: include/linux/can/dev.h F: include/linux/can/led.h +F: include/linux/can/length.h F: include/linux/can/platform/ F: include/linux/can/rx-offload.h F: include/uapi/linux/can/error.h diff --git a/drivers/net/can/dev/Makefile b/drivers/net/can/dev/Makefile index b5c6bb848d9d..5c647951e06d 100644 --- a/drivers/net/can/dev/Makefile +++ b/drivers/net/can/dev/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_CAN_DEV) += can-dev.o can-dev-y += bittiming.o can-dev-y += dev.o +can-dev-y += length.o can-dev-y += rx-offload.o can-dev-$(CONFIG_CAN_LEDS) += led.o diff --git a/drivers/net/can/dev/dev.c b/drivers/net/can/dev/dev.c index 1b3ab95b3fd1..3372e99d5db7 100644 --- a/drivers/net/can/dev/dev.c +++ b/drivers/net/can/dev/dev.c @@ -25,39 +25,6 @@ MODULE_DESCRIPTION(MOD_DESC); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Wolfgang Grandegger "); -/* CAN DLC to real data length conversion helpers */ - -static const u8 dlc2len[] = {0, 1, 2, 3, 4, 5, 6, 7, - 8, 12, 16, 20, 24, 32, 48, 64}; - -/* get data length from raw data length code (DLC) */ -u8 can_fd_dlc2len(u8 dlc) -{ - return dlc2len[dlc & 0x0F]; -} -EXPORT_SYMBOL_GPL(can_fd_dlc2len); - -static const u8 len2dlc[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, /* 0 - 8 */ - 9, 9, 9, 9, /* 9 - 12 */ - 10, 10, 10, 10, /* 13 - 16 */ - 11, 11, 11, 11, /* 17 - 20 */ - 12, 12, 12, 12, /* 21 - 24 */ - 13, 13, 13, 13, 13, 13, 13, 13, /* 25 - 32 */ - 14, 14, 14, 14, 14, 14, 14, 14, /* 33 - 40 */ - 14, 14, 14, 14, 14, 14, 14, 14, /* 41 - 48 */ - 15, 15, 15, 15, 15, 15, 15, 15, /* 49 - 56 */ - 15, 15, 15, 15, 15, 15, 15, 15}; /* 57 - 64 */ - -/* map the sanitized data length to an appropriate data length code */ -u8 can_fd_len2dlc(u8 len) -{ - if (unlikely(len > 64)) - return 0xF; - - return len2dlc[len]; -} -EXPORT_SYMBOL_GPL(can_fd_len2dlc); - static void can_update_state_error_stats(struct net_device *dev, enum can_state new_state) { diff --git a/drivers/net/can/dev/length.c b/drivers/net/can/dev/length.c new file mode 100644 index 000000000000..6fe18aa23ec9 --- /dev/null +++ b/drivers/net/can/dev/length.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2012, 2020 Oliver Hartkopp + */ + +#include + +/* CAN DLC to real data length conversion helpers */ + +static const u8 dlc2len[] = {0, 1, 2, 3, 4, 5, 6, 7, + 8, 12, 16, 20, 24, 32, 48, 64}; + +/* get data length from raw data length code (DLC) */ +u8 can_fd_dlc2len(u8 dlc) +{ + return dlc2len[dlc & 0x0F]; +} +EXPORT_SYMBOL_GPL(can_fd_dlc2len); + +static const u8 len2dlc[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, /* 0 - 8 */ + 9, 9, 9, 9, /* 9 - 12 */ + 10, 10, 10, 10, /* 13 - 16 */ + 11, 11, 11, 11, /* 17 - 20 */ + 12, 12, 12, 12, /* 21 - 24 */ + 13, 13, 13, 13, 13, 13, 13, 13, /* 25 - 32 */ + 14, 14, 14, 14, 14, 14, 14, 14, /* 33 - 40 */ + 14, 14, 14, 14, 14, 14, 14, 14, /* 41 - 48 */ + 15, 15, 15, 15, 15, 15, 15, 15, /* 49 - 56 */ + 15, 15, 15, 15, 15, 15, 15, 15}; /* 57 - 64 */ + +/* map the sanitized data length to an appropriate data length code */ +u8 can_fd_len2dlc(u8 len) +{ + if (unlikely(len > 64)) + return 0xF; + + return len2dlc[len]; +} +EXPORT_SYMBOL_GPL(can_fd_len2dlc); diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h index 054c3bed190b..d75fba1d030a 100644 --- a/include/linux/can/dev.h +++ b/include/linux/can/dev.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -83,15 +84,6 @@ struct can_priv { #endif }; -/* - * can_cc_dlc2len(value) - convert a given data length code (dlc) of a - * Classical CAN frame into a valid data length of max. 8 bytes. - * - * To be used in the CAN netdriver receive path to ensure conformance with - * ISO 11898-1 Chapter 8.4.2.3 (DLC field) - */ -#define can_cc_dlc2len(dlc) (min_t(u8, (dlc), CAN_MAX_DLEN)) - /* Check for outgoing skbs that have not been created by the CAN subsystem */ static inline bool can_skb_headroom_valid(struct net_device *dev, struct sk_buff *skb) @@ -156,31 +148,6 @@ static inline bool can_is_canfd_skb(const struct sk_buff *skb) return skb->len == CANFD_MTU; } -/* helper to get the data length code (DLC) for Classical CAN raw DLC access */ -static inline u8 can_get_cc_dlc(const struct can_frame *cf, const u32 ctrlmode) -{ - /* return len8_dlc as dlc value only if all conditions apply */ - if ((ctrlmode & CAN_CTRLMODE_CC_LEN8_DLC) && - (cf->len == CAN_MAX_DLEN) && - (cf->len8_dlc > CAN_MAX_DLEN && cf->len8_dlc <= CAN_MAX_RAW_DLC)) - return cf->len8_dlc; - - /* return the payload length as dlc value */ - return cf->len; -} - -/* helper to set len and len8_dlc value for Classical CAN raw DLC access */ -static inline void can_frame_set_cc_len(struct can_frame *cf, const u8 dlc, - const u32 ctrlmode) -{ - /* the caller already ensured that dlc is a value from 0 .. 15 */ - if (ctrlmode & CAN_CTRLMODE_CC_LEN8_DLC && dlc > CAN_MAX_DLEN) - cf->len8_dlc = dlc; - - /* limit the payload length 'len' to CAN_MAX_DLEN */ - cf->len = can_cc_dlc2len(dlc); -} - /* helper to define static CAN controller features at device creation time */ static inline void can_set_static_ctrlmode(struct net_device *dev, u32 static_mode) @@ -196,12 +163,6 @@ static inline void can_set_static_ctrlmode(struct net_device *dev, dev->mtu = CANFD_MTU; } -/* get data length from raw data length code (DLC) */ -u8 can_fd_dlc2len(u8 dlc); - -/* map the sanitized data length to an appropriate data length code */ -u8 can_fd_len2dlc(u8 len); - struct net_device *alloc_candev_mqs(int sizeof_priv, unsigned int echo_skb_max, unsigned int txqs, unsigned int rxqs); #define alloc_candev(sizeof_priv, echo_skb_max) \ diff --git a/include/linux/can/length.h b/include/linux/can/length.h new file mode 100644 index 000000000000..156b9d17969a --- /dev/null +++ b/include/linux/can/length.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2020 Oliver Hartkopp + */ + +#ifndef _CAN_LENGTH_H +#define _CAN_LENGTH_H + +/* + * can_cc_dlc2len(value) - convert a given data length code (dlc) of a + * Classical CAN frame into a valid data length of max. 8 bytes. + * + * To be used in the CAN netdriver receive path to ensure conformance with + * ISO 11898-1 Chapter 8.4.2.3 (DLC field) + */ +#define can_cc_dlc2len(dlc) (min_t(u8, (dlc), CAN_MAX_DLEN)) + +/* helper to get the data length code (DLC) for Classical CAN raw DLC access */ +static inline u8 can_get_cc_dlc(const struct can_frame *cf, const u32 ctrlmode) +{ + /* return len8_dlc as dlc value only if all conditions apply */ + if ((ctrlmode & CAN_CTRLMODE_CC_LEN8_DLC) && + (cf->len == CAN_MAX_DLEN) && + (cf->len8_dlc > CAN_MAX_DLEN && cf->len8_dlc <= CAN_MAX_RAW_DLC)) + return cf->len8_dlc; + + /* return the payload length as dlc value */ + return cf->len; +} + +/* helper to set len and len8_dlc value for Classical CAN raw DLC access */ +static inline void can_frame_set_cc_len(struct can_frame *cf, const u8 dlc, + const u32 ctrlmode) +{ + /* the caller already ensured that dlc is a value from 0 .. 15 */ + if (ctrlmode & CAN_CTRLMODE_CC_LEN8_DLC && dlc > CAN_MAX_DLEN) + cf->len8_dlc = dlc; + + /* limit the payload length 'len' to CAN_MAX_DLEN */ + cf->len = can_cc_dlc2len(dlc); +} + +/* get data length from raw data length code (DLC) */ +u8 can_fd_dlc2len(u8 dlc); + +/* map the sanitized data length to an appropriate data length code */ +u8 can_fd_len2dlc(u8 len); + +#endif /* !_CAN_LENGTH_H */ -- cgit v1.2.3 From 18f2dbfd2232212f53af9d249682f13a8335d54f Mon Sep 17 00:00:00 2001 From: Marc Kleine-Budde Date: Mon, 11 Jan 2021 15:19:20 +0100 Subject: can: dev: move skb related into seperate file This patch moves the skb related code of the CAN device infrastructure into a separate file. Reviewed-by: Vincent Mailhol Link: https://lore.kernel.org/r/20210111141930.693847-6-mkl@pengutronix.de Signed-off-by: Marc Kleine-Budde --- drivers/net/can/dev/Makefile | 1 + drivers/net/can/dev/dev.c | 213 ----------------------------------------- drivers/net/can/dev/skb.c | 220 +++++++++++++++++++++++++++++++++++++++++++ include/linux/can/dev.h | 76 --------------- include/linux/can/skb.h | 77 +++++++++++++++ 5 files changed, 298 insertions(+), 289 deletions(-) create mode 100644 drivers/net/can/dev/skb.c (limited to 'include/linux') diff --git a/drivers/net/can/dev/Makefile b/drivers/net/can/dev/Makefile index 5c647951e06d..188bd53b113c 100644 --- a/drivers/net/can/dev/Makefile +++ b/drivers/net/can/dev/Makefile @@ -5,5 +5,6 @@ can-dev-y += bittiming.o can-dev-y += dev.o can-dev-y += length.o can-dev-y += rx-offload.o +can-dev-y += skb.o can-dev-$(CONFIG_CAN_LEDS) += led.o diff --git a/drivers/net/can/dev/dev.c b/drivers/net/can/dev/dev.c index 3372e99d5db7..fe71ff9ef8c9 100644 --- a/drivers/net/can/dev/dev.c +++ b/drivers/net/can/dev/dev.c @@ -132,149 +132,6 @@ void can_change_state(struct net_device *dev, struct can_frame *cf, } EXPORT_SYMBOL_GPL(can_change_state); -/* Local echo of CAN messages - * - * CAN network devices *should* support a local echo functionality - * (see Documentation/networking/can.rst). To test the handling of CAN - * interfaces that do not support the local echo both driver types are - * implemented. In the case that the driver does not support the echo - * the IFF_ECHO remains clear in dev->flags. This causes the PF_CAN core - * to perform the echo as a fallback solution. - */ -static void can_flush_echo_skb(struct net_device *dev) -{ - struct can_priv *priv = netdev_priv(dev); - struct net_device_stats *stats = &dev->stats; - int i; - - for (i = 0; i < priv->echo_skb_max; i++) { - if (priv->echo_skb[i]) { - kfree_skb(priv->echo_skb[i]); - priv->echo_skb[i] = NULL; - stats->tx_dropped++; - stats->tx_aborted_errors++; - } - } -} - -/* Put the skb on the stack to be looped backed locally lateron - * - * The function is typically called in the start_xmit function - * of the device driver. The driver must protect access to - * priv->echo_skb, if necessary. - */ -int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, - unsigned int idx) -{ - struct can_priv *priv = netdev_priv(dev); - - BUG_ON(idx >= priv->echo_skb_max); - - /* check flag whether this packet has to be looped back */ - if (!(dev->flags & IFF_ECHO) || skb->pkt_type != PACKET_LOOPBACK || - (skb->protocol != htons(ETH_P_CAN) && - skb->protocol != htons(ETH_P_CANFD))) { - kfree_skb(skb); - return 0; - } - - if (!priv->echo_skb[idx]) { - skb = can_create_echo_skb(skb); - if (!skb) - return -ENOMEM; - - /* make settings for echo to reduce code in irq context */ - skb->pkt_type = PACKET_BROADCAST; - skb->ip_summed = CHECKSUM_UNNECESSARY; - skb->dev = dev; - - /* save this skb for tx interrupt echo handling */ - priv->echo_skb[idx] = skb; - } else { - /* locking problem with netif_stop_queue() ?? */ - netdev_err(dev, "%s: BUG! echo_skb %d is occupied!\n", __func__, idx); - kfree_skb(skb); - return -EBUSY; - } - - return 0; -} -EXPORT_SYMBOL_GPL(can_put_echo_skb); - -struct sk_buff * -__can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr) -{ - struct can_priv *priv = netdev_priv(dev); - - if (idx >= priv->echo_skb_max) { - netdev_err(dev, "%s: BUG! Trying to access can_priv::echo_skb out of bounds (%u/max %u)\n", - __func__, idx, priv->echo_skb_max); - return NULL; - } - - if (priv->echo_skb[idx]) { - /* Using "struct canfd_frame::len" for the frame - * length is supported on both CAN and CANFD frames. - */ - struct sk_buff *skb = priv->echo_skb[idx]; - struct canfd_frame *cf = (struct canfd_frame *)skb->data; - - /* get the real payload length for netdev statistics */ - if (cf->can_id & CAN_RTR_FLAG) - *len_ptr = 0; - else - *len_ptr = cf->len; - - priv->echo_skb[idx] = NULL; - - return skb; - } - - return NULL; -} - -/* Get the skb from the stack and loop it back locally - * - * The function is typically called when the TX done interrupt - * is handled in the device driver. The driver must protect - * access to priv->echo_skb, if necessary. - */ -unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx) -{ - struct sk_buff *skb; - u8 len; - - skb = __can_get_echo_skb(dev, idx, &len); - if (!skb) - return 0; - - skb_get(skb); - if (netif_rx(skb) == NET_RX_SUCCESS) - dev_consume_skb_any(skb); - else - dev_kfree_skb_any(skb); - - return len; -} -EXPORT_SYMBOL_GPL(can_get_echo_skb); - -/* Remove the skb from the stack and free it. - * - * The function is typically called when TX failed. - */ -void can_free_echo_skb(struct net_device *dev, unsigned int idx) -{ - struct can_priv *priv = netdev_priv(dev); - - BUG_ON(idx >= priv->echo_skb_max); - - if (priv->echo_skb[idx]) { - dev_kfree_skb_any(priv->echo_skb[idx]); - priv->echo_skb[idx] = NULL; - } -} -EXPORT_SYMBOL_GPL(can_free_echo_skb); - /* CAN device restart for bus-off recovery */ static void can_restart(struct net_device *dev) { @@ -379,76 +236,6 @@ static void can_setup(struct net_device *dev) dev->features = NETIF_F_HW_CSUM; } -struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf) -{ - struct sk_buff *skb; - - skb = netdev_alloc_skb(dev, sizeof(struct can_skb_priv) + - sizeof(struct can_frame)); - if (unlikely(!skb)) - return NULL; - - skb->protocol = htons(ETH_P_CAN); - skb->pkt_type = PACKET_BROADCAST; - skb->ip_summed = CHECKSUM_UNNECESSARY; - - skb_reset_mac_header(skb); - skb_reset_network_header(skb); - skb_reset_transport_header(skb); - - can_skb_reserve(skb); - can_skb_prv(skb)->ifindex = dev->ifindex; - can_skb_prv(skb)->skbcnt = 0; - - *cf = skb_put_zero(skb, sizeof(struct can_frame)); - - return skb; -} -EXPORT_SYMBOL_GPL(alloc_can_skb); - -struct sk_buff *alloc_canfd_skb(struct net_device *dev, - struct canfd_frame **cfd) -{ - struct sk_buff *skb; - - skb = netdev_alloc_skb(dev, sizeof(struct can_skb_priv) + - sizeof(struct canfd_frame)); - if (unlikely(!skb)) - return NULL; - - skb->protocol = htons(ETH_P_CANFD); - skb->pkt_type = PACKET_BROADCAST; - skb->ip_summed = CHECKSUM_UNNECESSARY; - - skb_reset_mac_header(skb); - skb_reset_network_header(skb); - skb_reset_transport_header(skb); - - can_skb_reserve(skb); - can_skb_prv(skb)->ifindex = dev->ifindex; - can_skb_prv(skb)->skbcnt = 0; - - *cfd = skb_put_zero(skb, sizeof(struct canfd_frame)); - - return skb; -} -EXPORT_SYMBOL_GPL(alloc_canfd_skb); - -struct sk_buff *alloc_can_err_skb(struct net_device *dev, struct can_frame **cf) -{ - struct sk_buff *skb; - - skb = alloc_can_skb(dev, cf); - if (unlikely(!skb)) - return NULL; - - (*cf)->can_id = CAN_ERR_FLAG; - (*cf)->len = CAN_ERR_DLC; - - return skb; -} -EXPORT_SYMBOL_GPL(alloc_can_err_skb); - /* Allocate and setup space for the CAN network device */ struct net_device *alloc_candev_mqs(int sizeof_priv, unsigned int echo_skb_max, unsigned int txqs, unsigned int rxqs) diff --git a/drivers/net/can/dev/skb.c b/drivers/net/can/dev/skb.c new file mode 100644 index 000000000000..26cd597ff771 --- /dev/null +++ b/drivers/net/can/dev/skb.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2005 Marc Kleine-Budde, Pengutronix + * Copyright (C) 2006 Andrey Volkov, Varma Electronics + * Copyright (C) 2008-2009 Wolfgang Grandegger + */ + +#include + +/* Local echo of CAN messages + * + * CAN network devices *should* support a local echo functionality + * (see Documentation/networking/can.rst). To test the handling of CAN + * interfaces that do not support the local echo both driver types are + * implemented. In the case that the driver does not support the echo + * the IFF_ECHO remains clear in dev->flags. This causes the PF_CAN core + * to perform the echo as a fallback solution. + */ +void can_flush_echo_skb(struct net_device *dev) +{ + struct can_priv *priv = netdev_priv(dev); + struct net_device_stats *stats = &dev->stats; + int i; + + for (i = 0; i < priv->echo_skb_max; i++) { + if (priv->echo_skb[i]) { + kfree_skb(priv->echo_skb[i]); + priv->echo_skb[i] = NULL; + stats->tx_dropped++; + stats->tx_aborted_errors++; + } + } +} + +/* Put the skb on the stack to be looped backed locally lateron + * + * The function is typically called in the start_xmit function + * of the device driver. The driver must protect access to + * priv->echo_skb, if necessary. + */ +int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, + unsigned int idx) +{ + struct can_priv *priv = netdev_priv(dev); + + BUG_ON(idx >= priv->echo_skb_max); + + /* check flag whether this packet has to be looped back */ + if (!(dev->flags & IFF_ECHO) || skb->pkt_type != PACKET_LOOPBACK || + (skb->protocol != htons(ETH_P_CAN) && + skb->protocol != htons(ETH_P_CANFD))) { + kfree_skb(skb); + return 0; + } + + if (!priv->echo_skb[idx]) { + skb = can_create_echo_skb(skb); + if (!skb) + return -ENOMEM; + + /* make settings for echo to reduce code in irq context */ + skb->pkt_type = PACKET_BROADCAST; + skb->ip_summed = CHECKSUM_UNNECESSARY; + skb->dev = dev; + + /* save this skb for tx interrupt echo handling */ + priv->echo_skb[idx] = skb; + } else { + /* locking problem with netif_stop_queue() ?? */ + netdev_err(dev, "%s: BUG! echo_skb %d is occupied!\n", __func__, idx); + kfree_skb(skb); + return -EBUSY; + } + + return 0; +} +EXPORT_SYMBOL_GPL(can_put_echo_skb); + +struct sk_buff * +__can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr) +{ + struct can_priv *priv = netdev_priv(dev); + + if (idx >= priv->echo_skb_max) { + netdev_err(dev, "%s: BUG! Trying to access can_priv::echo_skb out of bounds (%u/max %u)\n", + __func__, idx, priv->echo_skb_max); + return NULL; + } + + if (priv->echo_skb[idx]) { + /* Using "struct canfd_frame::len" for the frame + * length is supported on both CAN and CANFD frames. + */ + struct sk_buff *skb = priv->echo_skb[idx]; + struct canfd_frame *cf = (struct canfd_frame *)skb->data; + + /* get the real payload length for netdev statistics */ + if (cf->can_id & CAN_RTR_FLAG) + *len_ptr = 0; + else + *len_ptr = cf->len; + + priv->echo_skb[idx] = NULL; + + return skb; + } + + return NULL; +} + +/* Get the skb from the stack and loop it back locally + * + * The function is typically called when the TX done interrupt + * is handled in the device driver. The driver must protect + * access to priv->echo_skb, if necessary. + */ +unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx) +{ + struct sk_buff *skb; + u8 len; + + skb = __can_get_echo_skb(dev, idx, &len); + if (!skb) + return 0; + + skb_get(skb); + if (netif_rx(skb) == NET_RX_SUCCESS) + dev_consume_skb_any(skb); + else + dev_kfree_skb_any(skb); + + return len; +} +EXPORT_SYMBOL_GPL(can_get_echo_skb); + +/* Remove the skb from the stack and free it. + * + * The function is typically called when TX failed. + */ +void can_free_echo_skb(struct net_device *dev, unsigned int idx) +{ + struct can_priv *priv = netdev_priv(dev); + + BUG_ON(idx >= priv->echo_skb_max); + + if (priv->echo_skb[idx]) { + dev_kfree_skb_any(priv->echo_skb[idx]); + priv->echo_skb[idx] = NULL; + } +} +EXPORT_SYMBOL_GPL(can_free_echo_skb); + +struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf) +{ + struct sk_buff *skb; + + skb = netdev_alloc_skb(dev, sizeof(struct can_skb_priv) + + sizeof(struct can_frame)); + if (unlikely(!skb)) + return NULL; + + skb->protocol = htons(ETH_P_CAN); + skb->pkt_type = PACKET_BROADCAST; + skb->ip_summed = CHECKSUM_UNNECESSARY; + + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + + can_skb_reserve(skb); + can_skb_prv(skb)->ifindex = dev->ifindex; + can_skb_prv(skb)->skbcnt = 0; + + *cf = skb_put_zero(skb, sizeof(struct can_frame)); + + return skb; +} +EXPORT_SYMBOL_GPL(alloc_can_skb); + +struct sk_buff *alloc_canfd_skb(struct net_device *dev, + struct canfd_frame **cfd) +{ + struct sk_buff *skb; + + skb = netdev_alloc_skb(dev, sizeof(struct can_skb_priv) + + sizeof(struct canfd_frame)); + if (unlikely(!skb)) + return NULL; + + skb->protocol = htons(ETH_P_CANFD); + skb->pkt_type = PACKET_BROADCAST; + skb->ip_summed = CHECKSUM_UNNECESSARY; + + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + + can_skb_reserve(skb); + can_skb_prv(skb)->ifindex = dev->ifindex; + can_skb_prv(skb)->skbcnt = 0; + + *cfd = skb_put_zero(skb, sizeof(struct canfd_frame)); + + return skb; +} +EXPORT_SYMBOL_GPL(alloc_canfd_skb); + +struct sk_buff *alloc_can_err_skb(struct net_device *dev, struct can_frame **cf) +{ + struct sk_buff *skb; + + skb = alloc_can_skb(dev, cf); + if (unlikely(!skb)) + return NULL; + + (*cf)->can_id = CAN_ERR_FLAG; + (*cf)->len = CAN_ERR_DLC; + + return skb; +} +EXPORT_SYMBOL_GPL(alloc_can_err_skb); diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h index d75fba1d030a..4a26e128af7f 100644 --- a/include/linux/can/dev.h +++ b/include/linux/can/dev.h @@ -84,69 +84,6 @@ struct can_priv { #endif }; -/* Check for outgoing skbs that have not been created by the CAN subsystem */ -static inline bool can_skb_headroom_valid(struct net_device *dev, - struct sk_buff *skb) -{ - /* af_packet creates a headroom of HH_DATA_MOD bytes which is fine */ - if (WARN_ON_ONCE(skb_headroom(skb) < sizeof(struct can_skb_priv))) - return false; - - /* af_packet does not apply CAN skb specific settings */ - if (skb->ip_summed == CHECKSUM_NONE) { - /* init headroom */ - can_skb_prv(skb)->ifindex = dev->ifindex; - can_skb_prv(skb)->skbcnt = 0; - - skb->ip_summed = CHECKSUM_UNNECESSARY; - - /* perform proper loopback on capable devices */ - if (dev->flags & IFF_ECHO) - skb->pkt_type = PACKET_LOOPBACK; - else - skb->pkt_type = PACKET_HOST; - - skb_reset_mac_header(skb); - skb_reset_network_header(skb); - skb_reset_transport_header(skb); - } - - return true; -} - -/* Drop a given socketbuffer if it does not contain a valid CAN frame. */ -static inline bool can_dropped_invalid_skb(struct net_device *dev, - struct sk_buff *skb) -{ - const struct canfd_frame *cfd = (struct canfd_frame *)skb->data; - - if (skb->protocol == htons(ETH_P_CAN)) { - if (unlikely(skb->len != CAN_MTU || - cfd->len > CAN_MAX_DLEN)) - goto inval_skb; - } else if (skb->protocol == htons(ETH_P_CANFD)) { - if (unlikely(skb->len != CANFD_MTU || - cfd->len > CANFD_MAX_DLEN)) - goto inval_skb; - } else - goto inval_skb; - - if (!can_skb_headroom_valid(dev, skb)) - goto inval_skb; - - return false; - -inval_skb: - kfree_skb(skb); - dev->stats.tx_dropped++; - return true; -} - -static inline bool can_is_canfd_skb(const struct sk_buff *skb) -{ - /* the CAN specific type of skb is identified by its data length */ - return skb->len == CANFD_MTU; -} /* helper to define static CAN controller features at device creation time */ static inline void can_set_static_ctrlmode(struct net_device *dev, @@ -187,23 +124,10 @@ void can_bus_off(struct net_device *dev); void can_change_state(struct net_device *dev, struct can_frame *cf, enum can_state tx_state, enum can_state rx_state); -int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, - unsigned int idx); -struct sk_buff *__can_get_echo_skb(struct net_device *dev, unsigned int idx, - u8 *len_ptr); -unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx); -void can_free_echo_skb(struct net_device *dev, unsigned int idx); - #ifdef CONFIG_OF void of_can_transceiver(struct net_device *dev); #else static inline void of_can_transceiver(struct net_device *dev) { } #endif -struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf); -struct sk_buff *alloc_canfd_skb(struct net_device *dev, - struct canfd_frame **cfd); -struct sk_buff *alloc_can_err_skb(struct net_device *dev, - struct can_frame **cf); - #endif /* !_CAN_DEV_H */ diff --git a/include/linux/can/skb.h b/include/linux/can/skb.h index fc61cf4eff1c..c90ebbd3008c 100644 --- a/include/linux/can/skb.h +++ b/include/linux/can/skb.h @@ -16,6 +16,19 @@ #include #include +void can_flush_echo_skb(struct net_device *dev); +int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, + unsigned int idx); +struct sk_buff *__can_get_echo_skb(struct net_device *dev, unsigned int idx, + u8 *len_ptr); +unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx); +void can_free_echo_skb(struct net_device *dev, unsigned int idx); +struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf); +struct sk_buff *alloc_canfd_skb(struct net_device *dev, + struct canfd_frame **cfd); +struct sk_buff *alloc_can_err_skb(struct net_device *dev, + struct can_frame **cf); + /* * The struct can_skb_priv is used to transport additional information along * with the stored struct can(fd)_frame that can not be contained in existing @@ -74,4 +87,68 @@ static inline struct sk_buff *can_create_echo_skb(struct sk_buff *skb) return nskb; } +/* Check for outgoing skbs that have not been created by the CAN subsystem */ +static inline bool can_skb_headroom_valid(struct net_device *dev, + struct sk_buff *skb) +{ + /* af_packet creates a headroom of HH_DATA_MOD bytes which is fine */ + if (WARN_ON_ONCE(skb_headroom(skb) < sizeof(struct can_skb_priv))) + return false; + + /* af_packet does not apply CAN skb specific settings */ + if (skb->ip_summed == CHECKSUM_NONE) { + /* init headroom */ + can_skb_prv(skb)->ifindex = dev->ifindex; + can_skb_prv(skb)->skbcnt = 0; + + skb->ip_summed = CHECKSUM_UNNECESSARY; + + /* perform proper loopback on capable devices */ + if (dev->flags & IFF_ECHO) + skb->pkt_type = PACKET_LOOPBACK; + else + skb->pkt_type = PACKET_HOST; + + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + } + + return true; +} + +/* Drop a given socketbuffer if it does not contain a valid CAN frame. */ +static inline bool can_dropped_invalid_skb(struct net_device *dev, + struct sk_buff *skb) +{ + const struct canfd_frame *cfd = (struct canfd_frame *)skb->data; + + if (skb->protocol == htons(ETH_P_CAN)) { + if (unlikely(skb->len != CAN_MTU || + cfd->len > CAN_MAX_DLEN)) + goto inval_skb; + } else if (skb->protocol == htons(ETH_P_CANFD)) { + if (unlikely(skb->len != CANFD_MTU || + cfd->len > CANFD_MAX_DLEN)) + goto inval_skb; + } else + goto inval_skb; + + if (!can_skb_headroom_valid(dev, skb)) + goto inval_skb; + + return false; + +inval_skb: + kfree_skb(skb); + dev->stats.tx_dropped++; + return true; +} + +static inline bool can_is_canfd_skb(const struct sk_buff *skb) +{ + /* the CAN specific type of skb is identified by its data length */ + return skb->len == CANFD_MTU; +} + #endif /* !_CAN_SKB_H */ -- cgit v1.2.3 From 0a042c6ec991e4d33062ee52e9e23f615bc659f9 Mon Sep 17 00:00:00 2001 From: Marc Kleine-Budde Date: Mon, 11 Jan 2021 15:19:21 +0100 Subject: can: dev: move netlink related code into seperate file This patch moves the netlink related code of the CAN device infrastructure into a separate file. Reviewed-by: Vincent Mailhol Link: https://lore.kernel.org/r/20210111141930.693847-7-mkl@pengutronix.de Signed-off-by: Marc Kleine-Budde --- drivers/net/can/dev/Makefile | 1 + drivers/net/can/dev/dev.c | 370 +---------------------------------------- drivers/net/can/dev/netlink.c | 379 ++++++++++++++++++++++++++++++++++++++++++ include/linux/can/dev.h | 6 + 4 files changed, 389 insertions(+), 367 deletions(-) create mode 100644 drivers/net/can/dev/netlink.c (limited to 'include/linux') diff --git a/drivers/net/can/dev/Makefile b/drivers/net/can/dev/Makefile index 188bd53b113c..3e2e207861fc 100644 --- a/drivers/net/can/dev/Makefile +++ b/drivers/net/can/dev/Makefile @@ -4,6 +4,7 @@ obj-$(CONFIG_CAN_DEV) += can-dev.o can-dev-y += bittiming.o can-dev-y += dev.o can-dev-y += length.o +can-dev-y += netlink.o can-dev-y += rx-offload.o can-dev-y += skb.o diff --git a/drivers/net/can/dev/dev.c b/drivers/net/can/dev/dev.c index fe71ff9ef8c9..f580f0ac6497 100644 --- a/drivers/net/can/dev/dev.c +++ b/drivers/net/can/dev/dev.c @@ -14,10 +14,8 @@ #include #include #include -#include #include #include -#include #define MOD_DESC "CAN device driver interface" @@ -223,7 +221,7 @@ void can_bus_off(struct net_device *dev) } EXPORT_SYMBOL_GPL(can_bus_off); -static void can_setup(struct net_device *dev) +void can_setup(struct net_device *dev) { dev->type = ARPHRD_CAN; dev->mtu = CAN_MTU; @@ -399,368 +397,6 @@ void close_candev(struct net_device *dev) } EXPORT_SYMBOL_GPL(close_candev); -/* CAN netlink interface */ -static const struct nla_policy can_policy[IFLA_CAN_MAX + 1] = { - [IFLA_CAN_STATE] = { .type = NLA_U32 }, - [IFLA_CAN_CTRLMODE] = { .len = sizeof(struct can_ctrlmode) }, - [IFLA_CAN_RESTART_MS] = { .type = NLA_U32 }, - [IFLA_CAN_RESTART] = { .type = NLA_U32 }, - [IFLA_CAN_BITTIMING] = { .len = sizeof(struct can_bittiming) }, - [IFLA_CAN_BITTIMING_CONST] - = { .len = sizeof(struct can_bittiming_const) }, - [IFLA_CAN_CLOCK] = { .len = sizeof(struct can_clock) }, - [IFLA_CAN_BERR_COUNTER] = { .len = sizeof(struct can_berr_counter) }, - [IFLA_CAN_DATA_BITTIMING] - = { .len = sizeof(struct can_bittiming) }, - [IFLA_CAN_DATA_BITTIMING_CONST] - = { .len = sizeof(struct can_bittiming_const) }, - [IFLA_CAN_TERMINATION] = { .type = NLA_U16 }, -}; - -static int can_validate(struct nlattr *tb[], struct nlattr *data[], - struct netlink_ext_ack *extack) -{ - bool is_can_fd = false; - - /* Make sure that valid CAN FD configurations always consist of - * - nominal/arbitration bittiming - * - data bittiming - * - control mode with CAN_CTRLMODE_FD set - */ - - if (!data) - return 0; - - if (data[IFLA_CAN_CTRLMODE]) { - struct can_ctrlmode *cm = nla_data(data[IFLA_CAN_CTRLMODE]); - - is_can_fd = cm->flags & cm->mask & CAN_CTRLMODE_FD; - } - - if (is_can_fd) { - if (!data[IFLA_CAN_BITTIMING] || !data[IFLA_CAN_DATA_BITTIMING]) - return -EOPNOTSUPP; - } - - if (data[IFLA_CAN_DATA_BITTIMING]) { - if (!is_can_fd || !data[IFLA_CAN_BITTIMING]) - return -EOPNOTSUPP; - } - - return 0; -} - -static int can_changelink(struct net_device *dev, struct nlattr *tb[], - struct nlattr *data[], - struct netlink_ext_ack *extack) -{ - struct can_priv *priv = netdev_priv(dev); - int err; - - /* We need synchronization with dev->stop() */ - ASSERT_RTNL(); - - if (data[IFLA_CAN_BITTIMING]) { - struct can_bittiming bt; - - /* Do not allow changing bittiming while running */ - if (dev->flags & IFF_UP) - return -EBUSY; - - /* Calculate bittiming parameters based on - * bittiming_const if set, otherwise pass bitrate - * directly via do_set_bitrate(). Bail out if neither - * is given. - */ - if (!priv->bittiming_const && !priv->do_set_bittiming) - return -EOPNOTSUPP; - - memcpy(&bt, nla_data(data[IFLA_CAN_BITTIMING]), sizeof(bt)); - err = can_get_bittiming(dev, &bt, - priv->bittiming_const, - priv->bitrate_const, - priv->bitrate_const_cnt); - if (err) - return err; - - if (priv->bitrate_max && bt.bitrate > priv->bitrate_max) { - netdev_err(dev, "arbitration bitrate surpasses transceiver capabilities of %d bps\n", - priv->bitrate_max); - return -EINVAL; - } - - memcpy(&priv->bittiming, &bt, sizeof(bt)); - - if (priv->do_set_bittiming) { - /* Finally, set the bit-timing registers */ - err = priv->do_set_bittiming(dev); - if (err) - return err; - } - } - - if (data[IFLA_CAN_CTRLMODE]) { - struct can_ctrlmode *cm; - u32 ctrlstatic; - u32 maskedflags; - - /* Do not allow changing controller mode while running */ - if (dev->flags & IFF_UP) - return -EBUSY; - cm = nla_data(data[IFLA_CAN_CTRLMODE]); - ctrlstatic = priv->ctrlmode_static; - maskedflags = cm->flags & cm->mask; - - /* check whether provided bits are allowed to be passed */ - if (cm->mask & ~(priv->ctrlmode_supported | ctrlstatic)) - return -EOPNOTSUPP; - - /* do not check for static fd-non-iso if 'fd' is disabled */ - if (!(maskedflags & CAN_CTRLMODE_FD)) - ctrlstatic &= ~CAN_CTRLMODE_FD_NON_ISO; - - /* make sure static options are provided by configuration */ - if ((maskedflags & ctrlstatic) != ctrlstatic) - return -EOPNOTSUPP; - - /* clear bits to be modified and copy the flag values */ - priv->ctrlmode &= ~cm->mask; - priv->ctrlmode |= maskedflags; - - /* CAN_CTRLMODE_FD can only be set when driver supports FD */ - if (priv->ctrlmode & CAN_CTRLMODE_FD) - dev->mtu = CANFD_MTU; - else - dev->mtu = CAN_MTU; - } - - if (data[IFLA_CAN_RESTART_MS]) { - /* Do not allow changing restart delay while running */ - if (dev->flags & IFF_UP) - return -EBUSY; - priv->restart_ms = nla_get_u32(data[IFLA_CAN_RESTART_MS]); - } - - if (data[IFLA_CAN_RESTART]) { - /* Do not allow a restart while not running */ - if (!(dev->flags & IFF_UP)) - return -EINVAL; - err = can_restart_now(dev); - if (err) - return err; - } - - if (data[IFLA_CAN_DATA_BITTIMING]) { - struct can_bittiming dbt; - - /* Do not allow changing bittiming while running */ - if (dev->flags & IFF_UP) - return -EBUSY; - - /* Calculate bittiming parameters based on - * data_bittiming_const if set, otherwise pass bitrate - * directly via do_set_bitrate(). Bail out if neither - * is given. - */ - if (!priv->data_bittiming_const && !priv->do_set_data_bittiming) - return -EOPNOTSUPP; - - memcpy(&dbt, nla_data(data[IFLA_CAN_DATA_BITTIMING]), - sizeof(dbt)); - err = can_get_bittiming(dev, &dbt, - priv->data_bittiming_const, - priv->data_bitrate_const, - priv->data_bitrate_const_cnt); - if (err) - return err; - - if (priv->bitrate_max && dbt.bitrate > priv->bitrate_max) { - netdev_err(dev, "canfd data bitrate surpasses transceiver capabilities of %d bps\n", - priv->bitrate_max); - return -EINVAL; - } - - memcpy(&priv->data_bittiming, &dbt, sizeof(dbt)); - - if (priv->do_set_data_bittiming) { - /* Finally, set the bit-timing registers */ - err = priv->do_set_data_bittiming(dev); - if (err) - return err; - } - } - - if (data[IFLA_CAN_TERMINATION]) { - const u16 termval = nla_get_u16(data[IFLA_CAN_TERMINATION]); - const unsigned int num_term = priv->termination_const_cnt; - unsigned int i; - - if (!priv->do_set_termination) - return -EOPNOTSUPP; - - /* check whether given value is supported by the interface */ - for (i = 0; i < num_term; i++) { - if (termval == priv->termination_const[i]) - break; - } - if (i >= num_term) - return -EINVAL; - - /* Finally, set the termination value */ - err = priv->do_set_termination(dev, termval); - if (err) - return err; - - priv->termination = termval; - } - - return 0; -} - -static size_t can_get_size(const struct net_device *dev) -{ - struct can_priv *priv = netdev_priv(dev); - size_t size = 0; - - if (priv->bittiming.bitrate) /* IFLA_CAN_BITTIMING */ - size += nla_total_size(sizeof(struct can_bittiming)); - if (priv->bittiming_const) /* IFLA_CAN_BITTIMING_CONST */ - size += nla_total_size(sizeof(struct can_bittiming_const)); - size += nla_total_size(sizeof(struct can_clock)); /* IFLA_CAN_CLOCK */ - size += nla_total_size(sizeof(u32)); /* IFLA_CAN_STATE */ - size += nla_total_size(sizeof(struct can_ctrlmode)); /* IFLA_CAN_CTRLMODE */ - size += nla_total_size(sizeof(u32)); /* IFLA_CAN_RESTART_MS */ - if (priv->do_get_berr_counter) /* IFLA_CAN_BERR_COUNTER */ - size += nla_total_size(sizeof(struct can_berr_counter)); - if (priv->data_bittiming.bitrate) /* IFLA_CAN_DATA_BITTIMING */ - size += nla_total_size(sizeof(struct can_bittiming)); - if (priv->data_bittiming_const) /* IFLA_CAN_DATA_BITTIMING_CONST */ - size += nla_total_size(sizeof(struct can_bittiming_const)); - if (priv->termination_const) { - size += nla_total_size(sizeof(priv->termination)); /* IFLA_CAN_TERMINATION */ - size += nla_total_size(sizeof(*priv->termination_const) * /* IFLA_CAN_TERMINATION_CONST */ - priv->termination_const_cnt); - } - if (priv->bitrate_const) /* IFLA_CAN_BITRATE_CONST */ - size += nla_total_size(sizeof(*priv->bitrate_const) * - priv->bitrate_const_cnt); - if (priv->data_bitrate_const) /* IFLA_CAN_DATA_BITRATE_CONST */ - size += nla_total_size(sizeof(*priv->data_bitrate_const) * - priv->data_bitrate_const_cnt); - size += sizeof(priv->bitrate_max); /* IFLA_CAN_BITRATE_MAX */ - - return size; -} - -static int can_fill_info(struct sk_buff *skb, const struct net_device *dev) -{ - struct can_priv *priv = netdev_priv(dev); - struct can_ctrlmode cm = {.flags = priv->ctrlmode}; - struct can_berr_counter bec; - enum can_state state = priv->state; - - if (priv->do_get_state) - priv->do_get_state(dev, &state); - - if ((priv->bittiming.bitrate && - nla_put(skb, IFLA_CAN_BITTIMING, - sizeof(priv->bittiming), &priv->bittiming)) || - - (priv->bittiming_const && - nla_put(skb, IFLA_CAN_BITTIMING_CONST, - sizeof(*priv->bittiming_const), priv->bittiming_const)) || - - nla_put(skb, IFLA_CAN_CLOCK, sizeof(priv->clock), &priv->clock) || - nla_put_u32(skb, IFLA_CAN_STATE, state) || - nla_put(skb, IFLA_CAN_CTRLMODE, sizeof(cm), &cm) || - nla_put_u32(skb, IFLA_CAN_RESTART_MS, priv->restart_ms) || - - (priv->do_get_berr_counter && - !priv->do_get_berr_counter(dev, &bec) && - nla_put(skb, IFLA_CAN_BERR_COUNTER, sizeof(bec), &bec)) || - - (priv->data_bittiming.bitrate && - nla_put(skb, IFLA_CAN_DATA_BITTIMING, - sizeof(priv->data_bittiming), &priv->data_bittiming)) || - - (priv->data_bittiming_const && - nla_put(skb, IFLA_CAN_DATA_BITTIMING_CONST, - sizeof(*priv->data_bittiming_const), - priv->data_bittiming_const)) || - - (priv->termination_const && - (nla_put_u16(skb, IFLA_CAN_TERMINATION, priv->termination) || - nla_put(skb, IFLA_CAN_TERMINATION_CONST, - sizeof(*priv->termination_const) * - priv->termination_const_cnt, - priv->termination_const))) || - - (priv->bitrate_const && - nla_put(skb, IFLA_CAN_BITRATE_CONST, - sizeof(*priv->bitrate_const) * - priv->bitrate_const_cnt, - priv->bitrate_const)) || - - (priv->data_bitrate_const && - nla_put(skb, IFLA_CAN_DATA_BITRATE_CONST, - sizeof(*priv->data_bitrate_const) * - priv->data_bitrate_const_cnt, - priv->data_bitrate_const)) || - - (nla_put(skb, IFLA_CAN_BITRATE_MAX, - sizeof(priv->bitrate_max), - &priv->bitrate_max)) - ) - - return -EMSGSIZE; - - return 0; -} - -static size_t can_get_xstats_size(const struct net_device *dev) -{ - return sizeof(struct can_device_stats); -} - -static int can_fill_xstats(struct sk_buff *skb, const struct net_device *dev) -{ - struct can_priv *priv = netdev_priv(dev); - - if (nla_put(skb, IFLA_INFO_XSTATS, - sizeof(priv->can_stats), &priv->can_stats)) - goto nla_put_failure; - return 0; - -nla_put_failure: - return -EMSGSIZE; -} - -static int can_newlink(struct net *src_net, struct net_device *dev, - struct nlattr *tb[], struct nlattr *data[], - struct netlink_ext_ack *extack) -{ - return -EOPNOTSUPP; -} - -static void can_dellink(struct net_device *dev, struct list_head *head) -{ -} - -static struct rtnl_link_ops can_link_ops __read_mostly = { - .kind = "can", - .maxtype = IFLA_CAN_MAX, - .policy = can_policy, - .setup = can_setup, - .validate = can_validate, - .newlink = can_newlink, - .changelink = can_changelink, - .dellink = can_dellink, - .get_size = can_get_size, - .fill_info = can_fill_info, - .get_xstats_size = can_get_xstats_size, - .fill_xstats = can_fill_xstats, -}; - /* Register the CAN network device */ int register_candev(struct net_device *dev) { @@ -812,7 +448,7 @@ static __init int can_dev_init(void) can_led_notifier_init(); - err = rtnl_link_register(&can_link_ops); + err = can_netlink_register(); if (!err) pr_info(MOD_DESC "\n"); @@ -822,7 +458,7 @@ module_init(can_dev_init); static __exit void can_dev_exit(void) { - rtnl_link_unregister(&can_link_ops); + can_netlink_unregister(); can_led_notifier_exit(); } diff --git a/drivers/net/can/dev/netlink.c b/drivers/net/can/dev/netlink.c new file mode 100644 index 000000000000..3ae884cdf677 --- /dev/null +++ b/drivers/net/can/dev/netlink.c @@ -0,0 +1,379 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2005 Marc Kleine-Budde, Pengutronix + * Copyright (C) 2006 Andrey Volkov, Varma Electronics + * Copyright (C) 2008-2009 Wolfgang Grandegger + */ + +#include +#include + +static const struct nla_policy can_policy[IFLA_CAN_MAX + 1] = { + [IFLA_CAN_STATE] = { .type = NLA_U32 }, + [IFLA_CAN_CTRLMODE] = { .len = sizeof(struct can_ctrlmode) }, + [IFLA_CAN_RESTART_MS] = { .type = NLA_U32 }, + [IFLA_CAN_RESTART] = { .type = NLA_U32 }, + [IFLA_CAN_BITTIMING] = { .len = sizeof(struct can_bittiming) }, + [IFLA_CAN_BITTIMING_CONST] + = { .len = sizeof(struct can_bittiming_const) }, + [IFLA_CAN_CLOCK] = { .len = sizeof(struct can_clock) }, + [IFLA_CAN_BERR_COUNTER] = { .len = sizeof(struct can_berr_counter) }, + [IFLA_CAN_DATA_BITTIMING] + = { .len = sizeof(struct can_bittiming) }, + [IFLA_CAN_DATA_BITTIMING_CONST] + = { .len = sizeof(struct can_bittiming_const) }, + [IFLA_CAN_TERMINATION] = { .type = NLA_U16 }, +}; + +static int can_validate(struct nlattr *tb[], struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + bool is_can_fd = false; + + /* Make sure that valid CAN FD configurations always consist of + * - nominal/arbitration bittiming + * - data bittiming + * - control mode with CAN_CTRLMODE_FD set + */ + + if (!data) + return 0; + + if (data[IFLA_CAN_CTRLMODE]) { + struct can_ctrlmode *cm = nla_data(data[IFLA_CAN_CTRLMODE]); + + is_can_fd = cm->flags & cm->mask & CAN_CTRLMODE_FD; + } + + if (is_can_fd) { + if (!data[IFLA_CAN_BITTIMING] || !data[IFLA_CAN_DATA_BITTIMING]) + return -EOPNOTSUPP; + } + + if (data[IFLA_CAN_DATA_BITTIMING]) { + if (!is_can_fd || !data[IFLA_CAN_BITTIMING]) + return -EOPNOTSUPP; + } + + return 0; +} + +static int can_changelink(struct net_device *dev, struct nlattr *tb[], + struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + struct can_priv *priv = netdev_priv(dev); + int err; + + /* We need synchronization with dev->stop() */ + ASSERT_RTNL(); + + if (data[IFLA_CAN_BITTIMING]) { + struct can_bittiming bt; + + /* Do not allow changing bittiming while running */ + if (dev->flags & IFF_UP) + return -EBUSY; + + /* Calculate bittiming parameters based on + * bittiming_const if set, otherwise pass bitrate + * directly via do_set_bitrate(). Bail out if neither + * is given. + */ + if (!priv->bittiming_const && !priv->do_set_bittiming) + return -EOPNOTSUPP; + + memcpy(&bt, nla_data(data[IFLA_CAN_BITTIMING]), sizeof(bt)); + err = can_get_bittiming(dev, &bt, + priv->bittiming_const, + priv->bitrate_const, + priv->bitrate_const_cnt); + if (err) + return err; + + if (priv->bitrate_max && bt.bitrate > priv->bitrate_max) { + netdev_err(dev, "arbitration bitrate surpasses transceiver capabilities of %d bps\n", + priv->bitrate_max); + return -EINVAL; + } + + memcpy(&priv->bittiming, &bt, sizeof(bt)); + + if (priv->do_set_bittiming) { + /* Finally, set the bit-timing registers */ + err = priv->do_set_bittiming(dev); + if (err) + return err; + } + } + + if (data[IFLA_CAN_CTRLMODE]) { + struct can_ctrlmode *cm; + u32 ctrlstatic; + u32 maskedflags; + + /* Do not allow changing controller mode while running */ + if (dev->flags & IFF_UP) + return -EBUSY; + cm = nla_data(data[IFLA_CAN_CTRLMODE]); + ctrlstatic = priv->ctrlmode_static; + maskedflags = cm->flags & cm->mask; + + /* check whether provided bits are allowed to be passed */ + if (cm->mask & ~(priv->ctrlmode_supported | ctrlstatic)) + return -EOPNOTSUPP; + + /* do not check for static fd-non-iso if 'fd' is disabled */ + if (!(maskedflags & CAN_CTRLMODE_FD)) + ctrlstatic &= ~CAN_CTRLMODE_FD_NON_ISO; + + /* make sure static options are provided by configuration */ + if ((maskedflags & ctrlstatic) != ctrlstatic) + return -EOPNOTSUPP; + + /* clear bits to be modified and copy the flag values */ + priv->ctrlmode &= ~cm->mask; + priv->ctrlmode |= maskedflags; + + /* CAN_CTRLMODE_FD can only be set when driver supports FD */ + if (priv->ctrlmode & CAN_CTRLMODE_FD) + dev->mtu = CANFD_MTU; + else + dev->mtu = CAN_MTU; + } + + if (data[IFLA_CAN_RESTART_MS]) { + /* Do not allow changing restart delay while running */ + if (dev->flags & IFF_UP) + return -EBUSY; + priv->restart_ms = nla_get_u32(data[IFLA_CAN_RESTART_MS]); + } + + if (data[IFLA_CAN_RESTART]) { + /* Do not allow a restart while not running */ + if (!(dev->flags & IFF_UP)) + return -EINVAL; + err = can_restart_now(dev); + if (err) + return err; + } + + if (data[IFLA_CAN_DATA_BITTIMING]) { + struct can_bittiming dbt; + + /* Do not allow changing bittiming while running */ + if (dev->flags & IFF_UP) + return -EBUSY; + + /* Calculate bittiming parameters based on + * data_bittiming_const if set, otherwise pass bitrate + * directly via do_set_bitrate(). Bail out if neither + * is given. + */ + if (!priv->data_bittiming_const && !priv->do_set_data_bittiming) + return -EOPNOTSUPP; + + memcpy(&dbt, nla_data(data[IFLA_CAN_DATA_BITTIMING]), + sizeof(dbt)); + err = can_get_bittiming(dev, &dbt, + priv->data_bittiming_const, + priv->data_bitrate_const, + priv->data_bitrate_const_cnt); + if (err) + return err; + + if (priv->bitrate_max && dbt.bitrate > priv->bitrate_max) { + netdev_err(dev, "canfd data bitrate surpasses transceiver capabilities of %d bps\n", + priv->bitrate_max); + return -EINVAL; + } + + memcpy(&priv->data_bittiming, &dbt, sizeof(dbt)); + + if (priv->do_set_data_bittiming) { + /* Finally, set the bit-timing registers */ + err = priv->do_set_data_bittiming(dev); + if (err) + return err; + } + } + + if (data[IFLA_CAN_TERMINATION]) { + const u16 termval = nla_get_u16(data[IFLA_CAN_TERMINATION]); + const unsigned int num_term = priv->termination_const_cnt; + unsigned int i; + + if (!priv->do_set_termination) + return -EOPNOTSUPP; + + /* check whether given value is supported by the interface */ + for (i = 0; i < num_term; i++) { + if (termval == priv->termination_const[i]) + break; + } + if (i >= num_term) + return -EINVAL; + + /* Finally, set the termination value */ + err = priv->do_set_termination(dev, termval); + if (err) + return err; + + priv->termination = termval; + } + + return 0; +} + +static size_t can_get_size(const struct net_device *dev) +{ + struct can_priv *priv = netdev_priv(dev); + size_t size = 0; + + if (priv->bittiming.bitrate) /* IFLA_CAN_BITTIMING */ + size += nla_total_size(sizeof(struct can_bittiming)); + if (priv->bittiming_const) /* IFLA_CAN_BITTIMING_CONST */ + size += nla_total_size(sizeof(struct can_bittiming_const)); + size += nla_total_size(sizeof(struct can_clock)); /* IFLA_CAN_CLOCK */ + size += nla_total_size(sizeof(u32)); /* IFLA_CAN_STATE */ + size += nla_total_size(sizeof(struct can_ctrlmode)); /* IFLA_CAN_CTRLMODE */ + size += nla_total_size(sizeof(u32)); /* IFLA_CAN_RESTART_MS */ + if (priv->do_get_berr_counter) /* IFLA_CAN_BERR_COUNTER */ + size += nla_total_size(sizeof(struct can_berr_counter)); + if (priv->data_bittiming.bitrate) /* IFLA_CAN_DATA_BITTIMING */ + size += nla_total_size(sizeof(struct can_bittiming)); + if (priv->data_bittiming_const) /* IFLA_CAN_DATA_BITTIMING_CONST */ + size += nla_total_size(sizeof(struct can_bittiming_const)); + if (priv->termination_const) { + size += nla_total_size(sizeof(priv->termination)); /* IFLA_CAN_TERMINATION */ + size += nla_total_size(sizeof(*priv->termination_const) * /* IFLA_CAN_TERMINATION_CONST */ + priv->termination_const_cnt); + } + if (priv->bitrate_const) /* IFLA_CAN_BITRATE_CONST */ + size += nla_total_size(sizeof(*priv->bitrate_const) * + priv->bitrate_const_cnt); + if (priv->data_bitrate_const) /* IFLA_CAN_DATA_BITRATE_CONST */ + size += nla_total_size(sizeof(*priv->data_bitrate_const) * + priv->data_bitrate_const_cnt); + size += sizeof(priv->bitrate_max); /* IFLA_CAN_BITRATE_MAX */ + + return size; +} + +static int can_fill_info(struct sk_buff *skb, const struct net_device *dev) +{ + struct can_priv *priv = netdev_priv(dev); + struct can_ctrlmode cm = {.flags = priv->ctrlmode}; + struct can_berr_counter bec; + enum can_state state = priv->state; + + if (priv->do_get_state) + priv->do_get_state(dev, &state); + + if ((priv->bittiming.bitrate && + nla_put(skb, IFLA_CAN_BITTIMING, + sizeof(priv->bittiming), &priv->bittiming)) || + + (priv->bittiming_const && + nla_put(skb, IFLA_CAN_BITTIMING_CONST, + sizeof(*priv->bittiming_const), priv->bittiming_const)) || + + nla_put(skb, IFLA_CAN_CLOCK, sizeof(priv->clock), &priv->clock) || + nla_put_u32(skb, IFLA_CAN_STATE, state) || + nla_put(skb, IFLA_CAN_CTRLMODE, sizeof(cm), &cm) || + nla_put_u32(skb, IFLA_CAN_RESTART_MS, priv->restart_ms) || + + (priv->do_get_berr_counter && + !priv->do_get_berr_counter(dev, &bec) && + nla_put(skb, IFLA_CAN_BERR_COUNTER, sizeof(bec), &bec)) || + + (priv->data_bittiming.bitrate && + nla_put(skb, IFLA_CAN_DATA_BITTIMING, + sizeof(priv->data_bittiming), &priv->data_bittiming)) || + + (priv->data_bittiming_const && + nla_put(skb, IFLA_CAN_DATA_BITTIMING_CONST, + sizeof(*priv->data_bittiming_const), + priv->data_bittiming_const)) || + + (priv->termination_const && + (nla_put_u16(skb, IFLA_CAN_TERMINATION, priv->termination) || + nla_put(skb, IFLA_CAN_TERMINATION_CONST, + sizeof(*priv->termination_const) * + priv->termination_const_cnt, + priv->termination_const))) || + + (priv->bitrate_const && + nla_put(skb, IFLA_CAN_BITRATE_CONST, + sizeof(*priv->bitrate_const) * + priv->bitrate_const_cnt, + priv->bitrate_const)) || + + (priv->data_bitrate_const && + nla_put(skb, IFLA_CAN_DATA_BITRATE_CONST, + sizeof(*priv->data_bitrate_const) * + priv->data_bitrate_const_cnt, + priv->data_bitrate_const)) || + + (nla_put(skb, IFLA_CAN_BITRATE_MAX, + sizeof(priv->bitrate_max), + &priv->bitrate_max)) + ) + + return -EMSGSIZE; + + return 0; +} + +static size_t can_get_xstats_size(const struct net_device *dev) +{ + return sizeof(struct can_device_stats); +} + +static int can_fill_xstats(struct sk_buff *skb, const struct net_device *dev) +{ + struct can_priv *priv = netdev_priv(dev); + + if (nla_put(skb, IFLA_INFO_XSTATS, + sizeof(priv->can_stats), &priv->can_stats)) + goto nla_put_failure; + return 0; + +nla_put_failure: + return -EMSGSIZE; +} + +static int can_newlink(struct net *src_net, struct net_device *dev, + struct nlattr *tb[], struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + return -EOPNOTSUPP; +} + +static void can_dellink(struct net_device *dev, struct list_head *head) +{ +} + +struct rtnl_link_ops can_link_ops __read_mostly = { + .kind = "can", + .maxtype = IFLA_CAN_MAX, + .policy = can_policy, + .setup = can_setup, + .validate = can_validate, + .newlink = can_newlink, + .changelink = can_changelink, + .dellink = can_dellink, + .get_size = can_get_size, + .fill_info = can_fill_info, + .get_xstats_size = can_get_xstats_size, + .fill_xstats = can_fill_xstats, +}; + +int can_netlink_register(void) +{ + return rtnl_link_register(&can_link_ops); +} + +void can_netlink_unregister(void) +{ + rtnl_link_unregister(&can_link_ops); +} diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h index 4a26e128af7f..7faf6a37d5b2 100644 --- a/include/linux/can/dev.h +++ b/include/linux/can/dev.h @@ -100,6 +100,8 @@ static inline void can_set_static_ctrlmode(struct net_device *dev, dev->mtu = CANFD_MTU; } +void can_setup(struct net_device *dev); + struct net_device *alloc_candev_mqs(int sizeof_priv, unsigned int echo_skb_max, unsigned int txqs, unsigned int rxqs); #define alloc_candev(sizeof_priv, echo_skb_max) \ @@ -130,4 +132,8 @@ void of_can_transceiver(struct net_device *dev); static inline void of_can_transceiver(struct net_device *dev) { } #endif +extern struct rtnl_link_ops can_link_ops; +int can_netlink_register(void); +void can_netlink_unregister(void); + #endif /* !_CAN_DEV_H */ -- cgit v1.2.3 From 35192007d28d528aae74ddf5b26eaebf8e9a720c Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Fri, 18 Dec 2020 15:02:39 +0300 Subject: usb: phy: tegra: Support waking up from a low power mode Support programming of waking up from a low power mode by implementing the generic set_wakeup() callback of the USB PHY API. Tested-by: Matt Merhar Tested-by: Nicolas Chauvet Tested-by: Peter Geis Tested-by: Ion Agorria Acked-by: Thierry Reding Acked-by: Peter Chen Signed-off-by: Dmitry Osipenko Link: https://lore.kernel.org/r/20201218120246.7759-3-digetx@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/phy/phy-tegra-usb.c | 100 +++++++++++++++++++++++++++++++++----- include/linux/usb/tegra_usb_phy.h | 2 + 2 files changed, 91 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/phy/phy-tegra-usb.c b/drivers/usb/phy/phy-tegra-usb.c index 1296524e1bee..a48452a6172b 100644 --- a/drivers/usb/phy/phy-tegra-usb.c +++ b/drivers/usb/phy/phy-tegra-usb.c @@ -45,6 +45,7 @@ #define TEGRA_PORTSC1_RWC_BITS (PORT_CSC | PORT_PEC | PORT_OCC) #define USB_SUSP_CTRL 0x400 +#define USB_WAKE_ON_RESUME_EN BIT(2) #define USB_WAKE_ON_CNNT_EN_DEV BIT(3) #define USB_WAKE_ON_DISCON_EN_DEV BIT(4) #define USB_SUSP_CLR BIT(5) @@ -56,6 +57,15 @@ #define USB_SUSP_SET BIT(14) #define USB_WAKEUP_DEBOUNCE_COUNT(x) (((x) & 0x7) << 16) +#define USB_PHY_VBUS_SENSORS 0x404 +#define B_SESS_VLD_WAKEUP_EN BIT(6) +#define B_VBUS_VLD_WAKEUP_EN BIT(14) +#define A_SESS_VLD_WAKEUP_EN BIT(22) +#define A_VBUS_VLD_WAKEUP_EN BIT(30) + +#define USB_PHY_VBUS_WAKEUP_ID 0x408 +#define VBUS_WAKEUP_WAKEUP_EN BIT(30) + #define USB1_LEGACY_CTRL 0x410 #define USB1_NO_LEGACY_MODE BIT(0) #define USB1_VBUS_SENSE_CTL_MASK (3 << 1) @@ -334,6 +344,11 @@ static int utmip_pad_power_on(struct tegra_usb_phy *phy) writel_relaxed(val, base + UTMIP_BIAS_CFG0); } + if (phy->pad_wakeup) { + phy->pad_wakeup = false; + utmip_pad_count--; + } + spin_unlock(&utmip_pad_lock); clk_disable_unprepare(phy->pad_clk); @@ -359,6 +374,17 @@ static int utmip_pad_power_off(struct tegra_usb_phy *phy) goto ulock; } + /* + * In accordance to TRM, OTG and Bias pad circuits could be turned off + * to save power if wake is enabled, but the VBUS-change detection + * method is board-specific and these circuits may need to be enabled + * to generate wakeup event, hence we will just keep them both enabled. + */ + if (phy->wakeup_enabled) { + phy->pad_wakeup = true; + utmip_pad_count++; + } + if (--utmip_pad_count == 0) { val = readl_relaxed(base + UTMIP_BIAS_CFG0); val |= UTMIP_OTGPD | UTMIP_BIASPD; @@ -503,11 +529,24 @@ static int utmi_phy_power_on(struct tegra_usb_phy *phy) writel_relaxed(val, base + UTMIP_PLL_CFG1); } + val = readl_relaxed(base + USB_SUSP_CTRL); + val &= ~USB_WAKE_ON_RESUME_EN; + writel_relaxed(val, base + USB_SUSP_CTRL); + if (phy->mode == USB_DR_MODE_PERIPHERAL) { val = readl_relaxed(base + USB_SUSP_CTRL); val &= ~(USB_WAKE_ON_CNNT_EN_DEV | USB_WAKE_ON_DISCON_EN_DEV); writel_relaxed(val, base + USB_SUSP_CTRL); + val = readl_relaxed(base + USB_PHY_VBUS_WAKEUP_ID); + val &= ~VBUS_WAKEUP_WAKEUP_EN; + writel_relaxed(val, base + USB_PHY_VBUS_WAKEUP_ID); + + val = readl_relaxed(base + USB_PHY_VBUS_SENSORS); + val &= ~(A_VBUS_VLD_WAKEUP_EN | A_SESS_VLD_WAKEUP_EN); + val &= ~(B_VBUS_VLD_WAKEUP_EN | B_SESS_VLD_WAKEUP_EN); + writel_relaxed(val, base + USB_PHY_VBUS_SENSORS); + val = readl_relaxed(base + UTMIP_BAT_CHRG_CFG0); val &= ~UTMIP_PD_CHRG; writel_relaxed(val, base + UTMIP_BAT_CHRG_CFG0); @@ -605,31 +644,51 @@ static int utmi_phy_power_off(struct tegra_usb_phy *phy) utmi_phy_clk_disable(phy); - if (phy->mode == USB_DR_MODE_PERIPHERAL) { + /* PHY won't resume if reset is asserted */ + if (!phy->wakeup_enabled) { val = readl_relaxed(base + USB_SUSP_CTRL); - val &= ~USB_WAKEUP_DEBOUNCE_COUNT(~0); - val |= USB_WAKE_ON_CNNT_EN_DEV | USB_WAKEUP_DEBOUNCE_COUNT(5); + val |= UTMIP_RESET; writel_relaxed(val, base + USB_SUSP_CTRL); } - val = readl_relaxed(base + USB_SUSP_CTRL); - val |= UTMIP_RESET; - writel_relaxed(val, base + USB_SUSP_CTRL); - val = readl_relaxed(base + UTMIP_BAT_CHRG_CFG0); val |= UTMIP_PD_CHRG; writel_relaxed(val, base + UTMIP_BAT_CHRG_CFG0); - val = readl_relaxed(base + UTMIP_XCVR_CFG0); - val |= UTMIP_FORCE_PD_POWERDOWN | UTMIP_FORCE_PD2_POWERDOWN | - UTMIP_FORCE_PDZI_POWERDOWN; - writel_relaxed(val, base + UTMIP_XCVR_CFG0); + if (!phy->wakeup_enabled) { + val = readl_relaxed(base + UTMIP_XCVR_CFG0); + val |= UTMIP_FORCE_PD_POWERDOWN | UTMIP_FORCE_PD2_POWERDOWN | + UTMIP_FORCE_PDZI_POWERDOWN; + writel_relaxed(val, base + UTMIP_XCVR_CFG0); + } val = readl_relaxed(base + UTMIP_XCVR_CFG1); val |= UTMIP_FORCE_PDDISC_POWERDOWN | UTMIP_FORCE_PDCHRP_POWERDOWN | UTMIP_FORCE_PDDR_POWERDOWN; writel_relaxed(val, base + UTMIP_XCVR_CFG1); + if (phy->wakeup_enabled) { + val = readl_relaxed(base + USB_SUSP_CTRL); + val &= ~USB_WAKEUP_DEBOUNCE_COUNT(~0); + val |= USB_WAKEUP_DEBOUNCE_COUNT(5); + val |= USB_WAKE_ON_RESUME_EN; + writel_relaxed(val, base + USB_SUSP_CTRL); + + /* + * Ask VBUS sensor to generate wake event once cable is + * connected. + */ + if (phy->mode == USB_DR_MODE_PERIPHERAL) { + val = readl_relaxed(base + USB_PHY_VBUS_WAKEUP_ID); + val |= VBUS_WAKEUP_WAKEUP_EN; + writel_relaxed(val, base + USB_PHY_VBUS_WAKEUP_ID); + + val = readl_relaxed(base + USB_PHY_VBUS_SENSORS); + val |= A_VBUS_VLD_WAKEUP_EN; + writel_relaxed(val, base + USB_PHY_VBUS_SENSORS); + } + } + return utmip_pad_power_off(phy); } @@ -765,6 +824,15 @@ static int ulpi_phy_power_off(struct tegra_usb_phy *phy) usleep_range(5000, 6000); clk_disable_unprepare(phy->clk); + /* + * Wakeup currently unimplemented for ULPI, thus PHY needs to be + * force-resumed. + */ + if (WARN_ON_ONCE(phy->wakeup_enabled)) { + ulpi_phy_power_on(phy); + return -EOPNOTSUPP; + } + return 0; } @@ -827,6 +895,15 @@ static void tegra_usb_phy_shutdown(struct usb_phy *u_phy) phy->freq = NULL; } +static int tegra_usb_phy_set_wakeup(struct usb_phy *u_phy, bool enable) +{ + struct tegra_usb_phy *phy = to_tegra_usb_phy(u_phy); + + phy->wakeup_enabled = enable; + + return 0; +} + static int tegra_usb_phy_set_suspend(struct usb_phy *u_phy, int suspend) { struct tegra_usb_phy *phy = to_tegra_usb_phy(u_phy); @@ -1198,6 +1275,7 @@ static int tegra_usb_phy_probe(struct platform_device *pdev) tegra_phy->u_phy.dev = &pdev->dev; tegra_phy->u_phy.init = tegra_usb_phy_init; tegra_phy->u_phy.shutdown = tegra_usb_phy_shutdown; + tegra_phy->u_phy.set_wakeup = tegra_usb_phy_set_wakeup; tegra_phy->u_phy.set_suspend = tegra_usb_phy_set_suspend; platform_set_drvdata(pdev, tegra_phy); diff --git a/include/linux/usb/tegra_usb_phy.h b/include/linux/usb/tegra_usb_phy.h index c29d1b4c9381..fd1c9f6a4e37 100644 --- a/include/linux/usb/tegra_usb_phy.h +++ b/include/linux/usb/tegra_usb_phy.h @@ -79,6 +79,8 @@ struct tegra_usb_phy { bool is_ulpi_phy; struct gpio_desc *reset_gpio; struct reset_control *pad_rst; + bool wakeup_enabled; + bool pad_wakeup; bool powered_on; }; -- cgit v1.2.3 From fc53d5279094e38e6363506339772a7021da2df8 Mon Sep 17 00:00:00 2001 From: Peter Geis Date: Fri, 18 Dec 2020 15:02:42 +0300 Subject: usb: chipidea: tegra: Support host mode Add USB host mode to the Tegra HDRC driver. This allows us to benefit from support provided by the generic ChipIdea driver instead of duplicating the effort in a separate ehci-tegra driver. Tested-by: Matt Merhar Tested-by: Nicolas Chauvet Tested-by: Ion Agorria Acked-by: Thierry Reding Acked-by: Peter Chen Signed-off-by: Peter Geis Signed-off-by: Dmitry Osipenko Link: https://lore.kernel.org/r/20201218120246.7759-6-digetx@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/chipidea/Kconfig | 1 - drivers/usb/chipidea/ci_hdrc_tegra.c | 243 ++++++++++++++++++++++++++++++++++- drivers/usb/chipidea/core.c | 10 +- drivers/usb/chipidea/host.c | 104 ++++++++++++++- include/linux/usb/chipidea.h | 6 + 5 files changed, 356 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/chipidea/Kconfig b/drivers/usb/chipidea/Kconfig index 8685ead6ccc7..661818e8fed6 100644 --- a/drivers/usb/chipidea/Kconfig +++ b/drivers/usb/chipidea/Kconfig @@ -55,7 +55,6 @@ config USB_CHIPIDEA_GENERIC config USB_CHIPIDEA_TEGRA tristate "Enable Tegra USB glue driver" if EMBEDDED depends on OF - depends on USB_CHIPIDEA_UDC default USB_CHIPIDEA endif diff --git a/drivers/usb/chipidea/ci_hdrc_tegra.c b/drivers/usb/chipidea/ci_hdrc_tegra.c index d8efa80aa1c2..5fccdeeefc64 100644 --- a/drivers/usb/chipidea/ci_hdrc_tegra.c +++ b/drivers/usb/chipidea/ci_hdrc_tegra.c @@ -4,11 +4,18 @@ */ #include +#include #include #include #include +#include #include +#include +#include +#include + +#include "../host/ehci.h" #include "ci.h" @@ -16,20 +23,47 @@ struct tegra_usb { struct ci_hdrc_platform_data data; struct platform_device *dev; + const struct tegra_usb_soc_info *soc; struct usb_phy *phy; struct clk *clk; + + bool needs_double_reset; }; struct tegra_usb_soc_info { unsigned long flags; + unsigned int txfifothresh; + enum usb_dr_mode dr_mode; +}; + +static const struct tegra_usb_soc_info tegra20_ehci_soc_info = { + .flags = CI_HDRC_REQUIRES_ALIGNED_DMA | + CI_HDRC_OVERRIDE_PHY_CONTROL, + .dr_mode = USB_DR_MODE_HOST, + .txfifothresh = 10, +}; + +static const struct tegra_usb_soc_info tegra30_ehci_soc_info = { + .flags = CI_HDRC_REQUIRES_ALIGNED_DMA | + CI_HDRC_OVERRIDE_PHY_CONTROL, + .dr_mode = USB_DR_MODE_HOST, + .txfifothresh = 16, }; static const struct tegra_usb_soc_info tegra_udc_soc_info = { - .flags = CI_HDRC_REQUIRES_ALIGNED_DMA, + .flags = CI_HDRC_REQUIRES_ALIGNED_DMA | + CI_HDRC_OVERRIDE_PHY_CONTROL, + .dr_mode = USB_DR_MODE_UNKNOWN, }; static const struct of_device_id tegra_usb_of_match[] = { { + .compatible = "nvidia,tegra20-ehci", + .data = &tegra20_ehci_soc_info, + }, { + .compatible = "nvidia,tegra30-ehci", + .data = &tegra30_ehci_soc_info, + }, { .compatible = "nvidia,tegra20-udc", .data = &tegra_udc_soc_info, }, { @@ -47,6 +81,181 @@ static const struct of_device_id tegra_usb_of_match[] = { }; MODULE_DEVICE_TABLE(of, tegra_usb_of_match); +static int tegra_usb_reset_controller(struct device *dev) +{ + struct reset_control *rst, *rst_utmi; + struct device_node *phy_np; + int err; + + rst = devm_reset_control_get_shared(dev, "usb"); + if (IS_ERR(rst)) { + dev_err(dev, "can't get ehci reset: %pe\n", rst); + return PTR_ERR(rst); + } + + phy_np = of_parse_phandle(dev->of_node, "nvidia,phy", 0); + if (!phy_np) + return -ENOENT; + + /* + * The 1st USB controller contains some UTMI pad registers that are + * global for all the controllers on the chip. Those registers are + * also cleared when reset is asserted to the 1st controller. + */ + rst_utmi = of_reset_control_get_shared(phy_np, "utmi-pads"); + if (IS_ERR(rst_utmi)) { + dev_warn(dev, "can't get utmi-pads reset from the PHY\n"); + dev_warn(dev, "continuing, but please update your DT\n"); + } else { + /* + * PHY driver performs UTMI-pads reset in a case of a + * non-legacy DT. + */ + reset_control_put(rst_utmi); + } + + of_node_put(phy_np); + + /* reset control is shared, hence initialize it first */ + err = reset_control_deassert(rst); + if (err) + return err; + + err = reset_control_assert(rst); + if (err) + return err; + + udelay(1); + + err = reset_control_deassert(rst); + if (err) + return err; + + return 0; +} + +static int tegra_usb_notify_event(struct ci_hdrc *ci, unsigned int event) +{ + struct tegra_usb *usb = dev_get_drvdata(ci->dev->parent); + struct ehci_hcd *ehci; + + switch (event) { + case CI_HDRC_CONTROLLER_RESET_EVENT: + if (ci->hcd) { + ehci = hcd_to_ehci(ci->hcd); + ehci->has_tdi_phy_lpm = false; + ehci_writel(ehci, usb->soc->txfifothresh << 16, + &ehci->regs->txfill_tuning); + } + break; + } + + return 0; +} + +static int tegra_usb_internal_port_reset(struct ehci_hcd *ehci, + u32 __iomem *portsc_reg, + unsigned long *flags) +{ + u32 saved_usbintr, temp; + unsigned int i, tries; + int retval = 0; + + saved_usbintr = ehci_readl(ehci, &ehci->regs->intr_enable); + /* disable USB interrupt */ + ehci_writel(ehci, 0, &ehci->regs->intr_enable); + spin_unlock_irqrestore(&ehci->lock, *flags); + + /* + * Here we have to do Port Reset at most twice for + * Port Enable bit to be set. + */ + for (i = 0; i < 2; i++) { + temp = ehci_readl(ehci, portsc_reg); + temp |= PORT_RESET; + ehci_writel(ehci, temp, portsc_reg); + fsleep(10000); + temp &= ~PORT_RESET; + ehci_writel(ehci, temp, portsc_reg); + fsleep(1000); + tries = 100; + do { + fsleep(1000); + /* + * Up to this point, Port Enable bit is + * expected to be set after 2 ms waiting. + * USB1 usually takes extra 45 ms, for safety, + * we take 100 ms as timeout. + */ + temp = ehci_readl(ehci, portsc_reg); + } while (!(temp & PORT_PE) && tries--); + if (temp & PORT_PE) + break; + } + if (i == 2) + retval = -ETIMEDOUT; + + /* + * Clear Connect Status Change bit if it's set. + * We can't clear PORT_PEC. It will also cause PORT_PE to be cleared. + */ + if (temp & PORT_CSC) + ehci_writel(ehci, PORT_CSC, portsc_reg); + + /* + * Write to clear any interrupt status bits that might be set + * during port reset. + */ + temp = ehci_readl(ehci, &ehci->regs->status); + ehci_writel(ehci, temp, &ehci->regs->status); + + /* restore original interrupt-enable bits */ + spin_lock_irqsave(&ehci->lock, *flags); + ehci_writel(ehci, saved_usbintr, &ehci->regs->intr_enable); + + return retval; +} + +static int tegra_ehci_hub_control(struct ci_hdrc *ci, u16 typeReq, u16 wValue, + u16 wIndex, char *buf, u16 wLength, + bool *done, unsigned long *flags) +{ + struct tegra_usb *usb = dev_get_drvdata(ci->dev->parent); + struct ehci_hcd *ehci = hcd_to_ehci(ci->hcd); + u32 __iomem *status_reg; + int retval = 0; + + status_reg = &ehci->regs->port_status[(wIndex & 0xff) - 1]; + + switch (typeReq) { + case SetPortFeature: + if (wValue != USB_PORT_FEAT_RESET || !usb->needs_double_reset) + break; + + /* for USB1 port we need to issue Port Reset twice internally */ + retval = tegra_usb_internal_port_reset(ehci, status_reg, flags); + *done = true; + break; + } + + return retval; +} + +static void tegra_usb_enter_lpm(struct ci_hdrc *ci, bool enable) +{ + /* + * Touching any register which belongs to AHB clock domain will + * hang CPU if USB controller is put into low power mode because + * AHB USB clock is gated on Tegra in the LPM. + * + * Tegra PHY has a separate register for checking the clock status + * and usb_phy_set_suspend() takes care of gating/ungating the clocks + * and restoring the PHY state on Tegra. Hence DEVLC/PORTSC registers + * shouldn't be touched directly by the CI driver. + */ + usb_phy_set_suspend(ci->usb_phy, enable); +} + static int tegra_usb_probe(struct platform_device *pdev) { const struct tegra_usb_soc_info *soc; @@ -83,24 +292,49 @@ static int tegra_usb_probe(struct platform_device *pdev) return err; } + if (device_property_present(&pdev->dev, "nvidia,needs-double-reset")) + usb->needs_double_reset = true; + + err = tegra_usb_reset_controller(&pdev->dev); + if (err) { + dev_err(&pdev->dev, "failed to reset controller: %d\n", err); + goto fail_power_off; + } + + /* + * USB controller registers shouldn't be touched before PHY is + * initialized, otherwise CPU will hang because clocks are gated. + * PHY driver controls gating of internal USB clocks on Tegra. + */ + err = usb_phy_init(usb->phy); + if (err) + goto fail_power_off; + + platform_set_drvdata(pdev, usb); + /* setup and register ChipIdea HDRC device */ + usb->soc = soc; usb->data.name = "tegra-usb"; usb->data.flags = soc->flags; usb->data.usb_phy = usb->phy; + usb->data.dr_mode = soc->dr_mode; usb->data.capoffset = DEF_CAPOFFSET; + usb->data.enter_lpm = tegra_usb_enter_lpm; + usb->data.hub_control = tegra_ehci_hub_control; + usb->data.notify_event = tegra_usb_notify_event; usb->dev = ci_hdrc_add_device(&pdev->dev, pdev->resource, pdev->num_resources, &usb->data); if (IS_ERR(usb->dev)) { err = PTR_ERR(usb->dev); dev_err(&pdev->dev, "failed to add HDRC device: %d\n", err); - goto fail_power_off; + goto phy_shutdown; } - platform_set_drvdata(pdev, usb); - return 0; +phy_shutdown: + usb_phy_shutdown(usb->phy); fail_power_off: clk_disable_unprepare(usb->clk); return err; @@ -111,6 +345,7 @@ static int tegra_usb_remove(struct platform_device *pdev) struct tegra_usb *usb = platform_get_drvdata(pdev); ci_hdrc_remove_device(usb->dev); + usb_phy_shutdown(usb->phy); clk_disable_unprepare(usb->clk); return 0; diff --git a/drivers/usb/chipidea/core.c b/drivers/usb/chipidea/core.c index aa40e510b806..3f6c21406dbd 100644 --- a/drivers/usb/chipidea/core.c +++ b/drivers/usb/chipidea/core.c @@ -195,7 +195,7 @@ static void hw_wait_phy_stable(void) } /* The PHY enters/leaves low power mode */ -static void ci_hdrc_enter_lpm(struct ci_hdrc *ci, bool enable) +static void ci_hdrc_enter_lpm_common(struct ci_hdrc *ci, bool enable) { enum ci_hw_regs reg = ci->hw_bank.lpm ? OP_DEVLC : OP_PORTSC; bool lpm = !!(hw_read(ci, reg, PORTSC_PHCD(ci->hw_bank.lpm))); @@ -208,6 +208,11 @@ static void ci_hdrc_enter_lpm(struct ci_hdrc *ci, bool enable) 0); } +static void ci_hdrc_enter_lpm(struct ci_hdrc *ci, bool enable) +{ + return ci->platdata->enter_lpm(ci, enable); +} + static int hw_device_init(struct ci_hdrc *ci, void __iomem *base) { u32 reg; @@ -790,6 +795,9 @@ static int ci_get_platdata(struct device *dev, platdata->pins_device = p; } + if (!platdata->enter_lpm) + platdata->enter_lpm = ci_hdrc_enter_lpm_common; + return 0; } diff --git a/drivers/usb/chipidea/host.c b/drivers/usb/chipidea/host.c index 48e4a5ca1835..67247d2ac07a 100644 --- a/drivers/usb/chipidea/host.c +++ b/drivers/usb/chipidea/host.c @@ -29,6 +29,12 @@ struct ehci_ci_priv { bool enabled; }; +struct ci_hdrc_dma_aligned_buffer { + void *kmalloc_ptr; + void *old_xfer_buffer; + u8 data[0]; +}; + static int ehci_ci_portpower(struct usb_hcd *hcd, int portnum, bool enable) { struct ehci_hcd *ehci = hcd_to_ehci(hcd); @@ -160,14 +166,15 @@ static int host_start(struct ci_hdrc *ci) pinctrl_select_state(ci->platdata->pctl, ci->platdata->pins_host); + ci->hcd = hcd; + ret = usb_add_hcd(hcd, 0, 0); if (ret) { + ci->hcd = NULL; goto disable_reg; } else { struct usb_otg *otg = &ci->otg; - ci->hcd = hcd; - if (ci_otg_is_fsm_mode(ci)) { otg->host = &hcd->self; hcd->self.otg_port = 1; @@ -237,6 +244,7 @@ static int ci_ehci_hub_control( u32 temp; unsigned long flags; int retval = 0; + bool done = false; struct device *dev = hcd->self.controller; struct ci_hdrc *ci = dev_get_drvdata(dev); @@ -244,6 +252,13 @@ static int ci_ehci_hub_control( spin_lock_irqsave(&ehci->lock, flags); + if (ci->platdata->hub_control) { + retval = ci->platdata->hub_control(ci, typeReq, wValue, wIndex, + buf, wLength, &done, &flags); + if (done) + goto done; + } + if (typeReq == SetPortFeature && wValue == USB_PORT_FEAT_SUSPEND) { temp = ehci_readl(ehci, status_reg); if ((temp & PORT_PE) == 0 || (temp & PORT_RESET) != 0) { @@ -349,6 +364,86 @@ static int ci_ehci_bus_suspend(struct usb_hcd *hcd) return 0; } +static void ci_hdrc_free_dma_aligned_buffer(struct urb *urb) +{ + struct ci_hdrc_dma_aligned_buffer *temp; + size_t length; + + if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER)) + return; + + temp = container_of(urb->transfer_buffer, + struct ci_hdrc_dma_aligned_buffer, data); + + if (usb_urb_dir_in(urb)) { + if (usb_pipeisoc(urb->pipe)) + length = urb->transfer_buffer_length; + else + length = urb->actual_length; + + memcpy(temp->old_xfer_buffer, temp->data, length); + } + urb->transfer_buffer = temp->old_xfer_buffer; + kfree(temp->kmalloc_ptr); + + urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER; +} + +static int ci_hdrc_alloc_dma_aligned_buffer(struct urb *urb, gfp_t mem_flags) +{ + struct ci_hdrc_dma_aligned_buffer *temp, *kmalloc_ptr; + const unsigned int ci_hdrc_usb_dma_align = 32; + size_t kmalloc_size; + + if (urb->num_sgs || urb->sg || urb->transfer_buffer_length == 0 || + !((uintptr_t)urb->transfer_buffer & (ci_hdrc_usb_dma_align - 1))) + return 0; + + /* Allocate a buffer with enough padding for alignment */ + kmalloc_size = urb->transfer_buffer_length + + sizeof(struct ci_hdrc_dma_aligned_buffer) + + ci_hdrc_usb_dma_align - 1; + + kmalloc_ptr = kmalloc(kmalloc_size, mem_flags); + if (!kmalloc_ptr) + return -ENOMEM; + + /* Position our struct dma_aligned_buffer such that data is aligned */ + temp = PTR_ALIGN(kmalloc_ptr + 1, ci_hdrc_usb_dma_align) - 1; + temp->kmalloc_ptr = kmalloc_ptr; + temp->old_xfer_buffer = urb->transfer_buffer; + if (usb_urb_dir_out(urb)) + memcpy(temp->data, urb->transfer_buffer, + urb->transfer_buffer_length); + urb->transfer_buffer = temp->data; + + urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER; + + return 0; +} + +static int ci_hdrc_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb, + gfp_t mem_flags) +{ + int ret; + + ret = ci_hdrc_alloc_dma_aligned_buffer(urb, mem_flags); + if (ret) + return ret; + + ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags); + if (ret) + ci_hdrc_free_dma_aligned_buffer(urb); + + return ret; +} + +static void ci_hdrc_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb) +{ + usb_hcd_unmap_urb_for_dma(hcd, urb); + ci_hdrc_free_dma_aligned_buffer(urb); +} + int ci_hdrc_host_init(struct ci_hdrc *ci) { struct ci_role_driver *rdrv; @@ -366,6 +461,11 @@ int ci_hdrc_host_init(struct ci_hdrc *ci) rdrv->name = "host"; ci->roles[CI_ROLE_HOST] = rdrv; + if (ci->platdata->flags & CI_HDRC_REQUIRES_ALIGNED_DMA) { + ci_ehci_hc_driver.map_urb_for_dma = ci_hdrc_map_urb_for_dma; + ci_ehci_hc_driver.unmap_urb_for_dma = ci_hdrc_unmap_urb_for_dma; + } + return 0; } diff --git a/include/linux/usb/chipidea.h b/include/linux/usb/chipidea.h index 025b41687ce9..edf3342507f1 100644 --- a/include/linux/usb/chipidea.h +++ b/include/linux/usb/chipidea.h @@ -88,6 +88,12 @@ struct ci_hdrc_platform_data { struct pinctrl_state *pins_default; struct pinctrl_state *pins_host; struct pinctrl_state *pins_device; + + /* platform-specific hooks */ + int (*hub_control)(struct ci_hdrc *ci, u16 typeReq, u16 wValue, + u16 wIndex, char *buf, u16 wLength, + bool *done, unsigned long *flags); + void (*enter_lpm)(struct ci_hdrc *ci, bool enable); }; /* Default offset of capability registers */ -- cgit v1.2.3 From 838b00a2260ab6ec104a2a5696e619c19e8cd9be Mon Sep 17 00:00:00 2001 From: Paul Blakey Date: Mon, 11 Jan 2021 23:05:24 -0800 Subject: net/mlx5: Add HW definition of reg_c_preserve Add capability bit to test whether reg_c value is preserved on recirculation. Signed-off-by: Paul Blakey Signed-off-by: Maor Dickman Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- include/linux/mlx5/mlx5_ifc.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 442c0160caab..823411e288c0 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1278,7 +1278,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_a0[0x3]; u8 ece_support[0x1]; - u8 reserved_at_a4[0x7]; + u8 reserved_at_a4[0x5]; + u8 reg_c_preserve[0x1]; + u8 reserved_at_aa[0x1]; u8 log_max_srq[0x5]; u8 reserved_at_b0[0x1]; u8 uplink_follow[0x1]; -- cgit v1.2.3 From 99b7beb0431a4b9728023e9169ed15af678d2c7f Mon Sep 17 00:00:00 2001 From: Marc Kleine-Budde Date: Mon, 11 Jan 2021 15:19:24 +0100 Subject: can: length: canfd_sanitize_len(): add function to sanitize CAN-FD data length The data field in CAN-FD frames have specifig frame length (0, 1, 2, 3, 4, 5, 6, 7, 8, 12, 16, 20, 24, 32, 48, 64). This function "rounds" up a given length to the next valid CAN-FD frame length. Reviewed-by: Vincent Mailhol Link: https://lore.kernel.org/r/20210111141930.693847-10-mkl@pengutronix.de Signed-off-by: Marc Kleine-Budde --- include/linux/can/length.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/can/length.h b/include/linux/can/length.h index 156b9d17969a..b2313b2a0b02 100644 --- a/include/linux/can/length.h +++ b/include/linux/can/length.h @@ -45,4 +45,10 @@ u8 can_fd_dlc2len(u8 dlc); /* map the sanitized data length to an appropriate data length code */ u8 can_fd_len2dlc(u8 len); +/* map the data length to an appropriate data link layer length */ +static inline u8 canfd_sanitize_len(u8 len) +{ + return can_fd_dlc2len(can_fd_len2dlc(len)); +} + #endif /* !_CAN_LENGTH_H */ -- cgit v1.2.3 From 85d99c3e2a13d542445342a1383a686dadeaac3d Mon Sep 17 00:00:00 2001 From: Vincent Mailhol Date: Mon, 11 Jan 2021 15:19:25 +0100 Subject: can: length: can_skb_get_frame_len(): introduce function to get data length of frame in data link layer This patch adds the function can_skb_get_frame_len() which returns the length of a CAN frame on the data link layer, including Start-of-frame, Identifier, various other bits, the actual data, the CRC, the End-of-frame, the Inter frame spacing. Co-developed-by: Arunachalam Santhanam Signed-off-by: Arunachalam Santhanam Co-developed-by: Vincent Mailhol Signed-off-by: Vincent Mailhol Acked-by: Vincent Mailhol Reviewed-by: Vincent Mailhol Co-developed-by: Marc Kleine-Budde Link: https://lore.kernel.org/r/20210111141930.693847-11-mkl@pengutronix.de Signed-off-by: Marc Kleine-Budde --- drivers/net/can/dev/length.c | 50 ++++++++++++++++++ include/linux/can/length.h | 120 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/can/dev/length.c b/drivers/net/can/dev/length.c index 64673a8d1234..d35c4e82314d 100644 --- a/drivers/net/can/dev/length.c +++ b/drivers/net/can/dev/length.c @@ -38,3 +38,53 @@ u8 can_fd_len2dlc(u8 len) return len2dlc[len]; } EXPORT_SYMBOL_GPL(can_fd_len2dlc); + +/** + * can_skb_get_frame_len() - Calculate the CAN Frame length in bytes + * of a given skb. + * @skb: socket buffer of a CAN message. + * + * Do a rough calculation: bit stuffing is ignored and length in bits + * is rounded up to a length in bytes. + * + * Rationale: this function is to be used for the BQL functions + * (netdev_sent_queue() and netdev_completed_queue()) which expect a + * value in bytes. Just using skb->len is insufficient because it will + * return the constant value of CAN(FD)_MTU. Doing the bit stuffing + * calculation would be too expensive in term of computing resources + * for no noticeable gain. + * + * Remarks: The payload of CAN FD frames with BRS flag are sent at a + * different bitrate. Currently, the can-utils canbusload tool does + * not support CAN-FD yet and so we could not run any benchmark to + * measure the impact. There might be possible improvement here. + * + * Return: length in bytes. + */ +unsigned int can_skb_get_frame_len(const struct sk_buff *skb) +{ + const struct canfd_frame *cf = (const struct canfd_frame *)skb->data; + u8 len; + + if (can_is_canfd_skb(skb)) + len = canfd_sanitize_len(cf->len); + else if (cf->can_id & CAN_RTR_FLAG) + len = 0; + else + len = cf->len; + + if (can_is_canfd_skb(skb)) { + if (cf->can_id & CAN_EFF_FLAG) + len += CANFD_FRAME_OVERHEAD_EFF; + else + len += CANFD_FRAME_OVERHEAD_SFF; + } else { + if (cf->can_id & CAN_EFF_FLAG) + len += CAN_FRAME_OVERHEAD_EFF; + else + len += CAN_FRAME_OVERHEAD_SFF; + } + + return len; +} +EXPORT_SYMBOL_GPL(can_skb_get_frame_len); diff --git a/include/linux/can/length.h b/include/linux/can/length.h index b2313b2a0b02..6995092b774e 100644 --- a/include/linux/can/length.h +++ b/include/linux/can/length.h @@ -1,10 +1,127 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (C) 2020 Oliver Hartkopp + * Copyright (C) 2020 Marc Kleine-Budde */ #ifndef _CAN_LENGTH_H #define _CAN_LENGTH_H +/* + * Size of a Classical CAN Standard Frame + * + * Name of Field Bits + * --------------------------------------------------------- + * Start-of-frame 1 + * Identifier 11 + * Remote transmission request (RTR) 1 + * Identifier extension bit (IDE) 1 + * Reserved bit (r0) 1 + * Data length code (DLC) 4 + * Data field 0...64 + * CRC 15 + * CRC delimiter 1 + * ACK slot 1 + * ACK delimiter 1 + * End-of-frame (EOF) 7 + * Inter frame spacing 3 + * + * rounded up and ignoring bitstuffing + */ +#define CAN_FRAME_OVERHEAD_SFF DIV_ROUND_UP(47, 8) + +/* + * Size of a Classical CAN Extended Frame + * + * Name of Field Bits + * --------------------------------------------------------- + * Start-of-frame 1 + * Identifier A 11 + * Substitute remote request (SRR) 1 + * Identifier extension bit (IDE) 1 + * Identifier B 18 + * Remote transmission request (RTR) 1 + * Reserved bits (r1, r0) 2 + * Data length code (DLC) 4 + * Data field 0...64 + * CRC 15 + * CRC delimiter 1 + * ACK slot 1 + * ACK delimiter 1 + * End-of-frame (EOF) 7 + * Inter frame spacing 3 + * + * rounded up and ignoring bitstuffing + */ +#define CAN_FRAME_OVERHEAD_EFF DIV_ROUND_UP(67, 8) + +/* + * Size of a CAN-FD Standard Frame + * + * Name of Field Bits + * --------------------------------------------------------- + * Start-of-frame 1 + * Identifier 11 + * Reserved bit (r1) 1 + * Identifier extension bit (IDE) 1 + * Flexible data rate format (FDF) 1 + * Reserved bit (r0) 1 + * Bit Rate Switch (BRS) 1 + * Error Status Indicator (ESI) 1 + * Data length code (DLC) 4 + * Data field 0...512 + * Stuff Bit Count (SBC) 0...16: 4 20...64:5 + * CRC 0...16: 17 20...64:21 + * CRC delimiter (CD) 1 + * ACK slot (AS) 1 + * ACK delimiter (AD) 1 + * End-of-frame (EOF) 7 + * Inter frame spacing 3 + * + * assuming CRC21, rounded up and ignoring bitstuffing + */ +#define CANFD_FRAME_OVERHEAD_SFF DIV_ROUND_UP(61, 8) + +/* + * Size of a CAN-FD Extended Frame + * + * Name of Field Bits + * --------------------------------------------------------- + * Start-of-frame 1 + * Identifier A 11 + * Substitute remote request (SRR) 1 + * Identifier extension bit (IDE) 1 + * Identifier B 18 + * Reserved bit (r1) 1 + * Flexible data rate format (FDF) 1 + * Reserved bit (r0) 1 + * Bit Rate Switch (BRS) 1 + * Error Status Indicator (ESI) 1 + * Data length code (DLC) 4 + * Data field 0...512 + * Stuff Bit Count (SBC) 0...16: 4 20...64:5 + * CRC 0...16: 17 20...64:21 + * CRC delimiter (CD) 1 + * ACK slot (AS) 1 + * ACK delimiter (AD) 1 + * End-of-frame (EOF) 7 + * Inter frame spacing 3 + * + * assuming CRC21, rounded up and ignoring bitstuffing + */ +#define CANFD_FRAME_OVERHEAD_EFF DIV_ROUND_UP(80, 8) + +/* + * Maximum size of a Classical CAN frame + * (rounded up and ignoring bitstuffing) + */ +#define CAN_FRAME_LEN_MAX (CAN_FRAME_OVERHEAD_EFF + CAN_MAX_DLEN) + +/* + * Maximum size of a CAN-FD frame + * (rounded up and ignoring bitstuffing) + */ +#define CANFD_FRAME_LEN_MAX (CANFD_FRAME_OVERHEAD_EFF + CANFD_MAX_DLEN) + /* * can_cc_dlc2len(value) - convert a given data length code (dlc) of a * Classical CAN frame into a valid data length of max. 8 bytes. @@ -45,6 +162,9 @@ u8 can_fd_dlc2len(u8 dlc); /* map the sanitized data length to an appropriate data length code */ u8 can_fd_len2dlc(u8 len); +/* calculate the CAN Frame length in bytes of a given skb */ +unsigned int can_skb_get_frame_len(const struct sk_buff *skb); + /* map the data length to an appropriate data link layer length */ static inline u8 canfd_sanitize_len(u8 len) { -- cgit v1.2.3 From f0ef72febc9a6a569d92cdf6c7996015dfa8e8bb Mon Sep 17 00:00:00 2001 From: Marc Kleine-Budde Date: Mon, 11 Jan 2021 15:19:26 +0100 Subject: can: dev: extend struct can_skb_priv to hold CAN frame length In order to implement byte queue limits (bql) in CAN drivers, the length of the CAN frame needs to be passed into the networking stack after queueing and after transmission completion. To avoid to calculate this length twice, extend the struct can_skb_priv to hold the length of the CAN frame and extend __can_get_echo_skb() to return that value. Reviewed-by: Vincent Mailhol Link: https://lore.kernel.org/r/20210111141930.693847-12-mkl@pengutronix.de Signed-off-by: Marc Kleine-Budde --- drivers/net/can/dev/rx-offload.c | 2 +- drivers/net/can/dev/skb.c | 9 +++++++-- include/linux/can/skb.h | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/can/dev/rx-offload.c b/drivers/net/can/dev/rx-offload.c index 3c1912c0430b..6a26b5282df1 100644 --- a/drivers/net/can/dev/rx-offload.c +++ b/drivers/net/can/dev/rx-offload.c @@ -271,7 +271,7 @@ unsigned int can_rx_offload_get_echo_skb(struct can_rx_offload *offload, u8 len; int err; - skb = __can_get_echo_skb(dev, idx, &len); + skb = __can_get_echo_skb(dev, idx, &len, NULL); if (!skb) return 0; diff --git a/drivers/net/can/dev/skb.c b/drivers/net/can/dev/skb.c index 26cd597ff771..24f782a23409 100644 --- a/drivers/net/can/dev/skb.c +++ b/drivers/net/can/dev/skb.c @@ -76,7 +76,8 @@ int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, EXPORT_SYMBOL_GPL(can_put_echo_skb); struct sk_buff * -__can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr) +__can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr, + unsigned int *frame_len_ptr) { struct can_priv *priv = netdev_priv(dev); @@ -91,6 +92,7 @@ __can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr) * length is supported on both CAN and CANFD frames. */ struct sk_buff *skb = priv->echo_skb[idx]; + struct can_skb_priv *can_skb_priv = can_skb_prv(skb); struct canfd_frame *cf = (struct canfd_frame *)skb->data; /* get the real payload length for netdev statistics */ @@ -99,6 +101,9 @@ __can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr) else *len_ptr = cf->len; + if (frame_len_ptr) + *frame_len_ptr = can_skb_priv->frame_len; + priv->echo_skb[idx] = NULL; return skb; @@ -118,7 +123,7 @@ unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx) struct sk_buff *skb; u8 len; - skb = __can_get_echo_skb(dev, idx, &len); + skb = __can_get_echo_skb(dev, idx, &len, NULL); if (!skb) return 0; diff --git a/include/linux/can/skb.h b/include/linux/can/skb.h index c90ebbd3008c..5db9da30843c 100644 --- a/include/linux/can/skb.h +++ b/include/linux/can/skb.h @@ -20,7 +20,7 @@ void can_flush_echo_skb(struct net_device *dev); int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, unsigned int idx); struct sk_buff *__can_get_echo_skb(struct net_device *dev, unsigned int idx, - u8 *len_ptr); + u8 *len_ptr, unsigned int *frame_len_ptr); unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx); void can_free_echo_skb(struct net_device *dev, unsigned int idx); struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf); @@ -42,11 +42,13 @@ struct sk_buff *alloc_can_err_skb(struct net_device *dev, * struct can_skb_priv - private additional data inside CAN sk_buffs * @ifindex: ifindex of the first interface the CAN frame appeared on * @skbcnt: atomic counter to have an unique id together with skb pointer + * @frame_len: length of CAN frame in data link layer * @cf: align to the following CAN frame at skb->data */ struct can_skb_priv { int ifindex; int skbcnt; + unsigned int frame_len; struct can_frame cf[]; }; -- cgit v1.2.3 From 1dcb6e57db833419483d0df2d956b1cc2a802683 Mon Sep 17 00:00:00 2001 From: Vincent Mailhol Date: Mon, 11 Jan 2021 15:19:27 +0100 Subject: can: dev: can_put_echo_skb(): extend to handle frame_len Add a frame_len argument to can_put_echo_skb() which is used to save length of the CAN frame into field frame_len of struct can_skb_priv so that it can be later used after transmission completion. Convert all users of this function, too. Drivers which implement BQL call can_put_echo_skb() with the output of can_skb_get_frame_len(skb) and drivers which do not simply pass zero as an input (in the same way that NULL would be given to can_get_echo_skb()). This way, we have a nice symmetry between the two echo functions. Link: https://lore.kernel.org/r/20210111061335.39983-1-mailhol.vincent@wanadoo.fr Signed-off-by: Marc Kleine-Budde Reviewed-by: Vincent Mailhol Link: https://lore.kernel.org/r/20210111141930.693847-13-mkl@pengutronix.de Signed-off-by: Vincent Mailhol --- drivers/net/can/at91_can.c | 2 +- drivers/net/can/c_can/c_can.c | 2 +- drivers/net/can/cc770/cc770.c | 2 +- drivers/net/can/dev/skb.c | 5 ++++- drivers/net/can/flexcan.c | 2 +- drivers/net/can/grcan.c | 2 +- drivers/net/can/ifi_canfd/ifi_canfd.c | 2 +- drivers/net/can/kvaser_pciefd.c | 2 +- drivers/net/can/m_can/m_can.c | 4 ++-- drivers/net/can/mscan/mscan.c | 2 +- drivers/net/can/pch_can.c | 2 +- drivers/net/can/peak_canfd/peak_canfd.c | 2 +- drivers/net/can/rcar/rcar_can.c | 2 +- drivers/net/can/rcar/rcar_canfd.c | 2 +- drivers/net/can/sja1000/sja1000.c | 2 +- drivers/net/can/softing/softing_main.c | 2 +- drivers/net/can/spi/hi311x.c | 2 +- drivers/net/can/spi/mcp251x.c | 2 +- drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c | 2 +- drivers/net/can/sun4i_can.c | 2 +- drivers/net/can/ti_hecc.c | 2 +- drivers/net/can/usb/ems_usb.c | 2 +- drivers/net/can/usb/esd_usb2.c | 2 +- drivers/net/can/usb/gs_usb.c | 2 +- drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c | 2 +- drivers/net/can/usb/mcba_usb.c | 2 +- drivers/net/can/usb/peak_usb/pcan_usb_core.c | 2 +- drivers/net/can/usb/ucan.c | 2 +- drivers/net/can/usb/usb_8dev.c | 2 +- drivers/net/can/xilinx_can.c | 4 ++-- include/linux/can/skb.h | 2 +- 31 files changed, 36 insertions(+), 33 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/can/at91_can.c b/drivers/net/can/at91_can.c index 5284f0ab3b06..90b223a80ed4 100644 --- a/drivers/net/can/at91_can.c +++ b/drivers/net/can/at91_can.c @@ -484,7 +484,7 @@ static netdev_tx_t at91_start_xmit(struct sk_buff *skb, struct net_device *dev) stats->tx_bytes += cf->len; /* _NOTE_: subtract AT91_MB_TX_FIRST offset from mb! */ - can_put_echo_skb(skb, dev, mb - get_mb_tx_first(priv)); + can_put_echo_skb(skb, dev, mb - get_mb_tx_first(priv), 0); /* * we have to stop the queue and deliver all messages in case diff --git a/drivers/net/can/c_can/c_can.c b/drivers/net/can/c_can/c_can.c index 63f48b016ecd..13638954a25c 100644 --- a/drivers/net/can/c_can/c_can.c +++ b/drivers/net/can/c_can/c_can.c @@ -476,7 +476,7 @@ static netdev_tx_t c_can_start_xmit(struct sk_buff *skb, */ c_can_setup_tx_object(dev, IF_TX, frame, idx); priv->dlc[idx] = frame->len; - can_put_echo_skb(skb, dev, idx); + can_put_echo_skb(skb, dev, idx, 0); /* Update the active bits */ atomic_add((1 << idx), &priv->tx_active); diff --git a/drivers/net/can/cc770/cc770.c b/drivers/net/can/cc770/cc770.c index 8d9f332c35e0..e53ca338368a 100644 --- a/drivers/net/can/cc770/cc770.c +++ b/drivers/net/can/cc770/cc770.c @@ -702,7 +702,7 @@ static void cc770_tx_interrupt(struct net_device *dev, unsigned int o) stats->tx_bytes += cf->len; stats->tx_packets++; - can_put_echo_skb(priv->tx_skb, dev, 0); + can_put_echo_skb(priv->tx_skb, dev, 0, 0); can_get_echo_skb(dev, 0); priv->tx_skb = NULL; diff --git a/drivers/net/can/dev/skb.c b/drivers/net/can/dev/skb.c index 24f782a23409..c184b4dce19e 100644 --- a/drivers/net/can/dev/skb.c +++ b/drivers/net/can/dev/skb.c @@ -38,7 +38,7 @@ void can_flush_echo_skb(struct net_device *dev) * priv->echo_skb, if necessary. */ int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, - unsigned int idx) + unsigned int idx, unsigned int frame_len) { struct can_priv *priv = netdev_priv(dev); @@ -62,6 +62,9 @@ int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, skb->ip_summed = CHECKSUM_UNNECESSARY; skb->dev = dev; + /* save frame_len to reuse it when transmission is completed */ + can_skb_prv(skb)->frame_len = frame_len; + /* save this skb for tx interrupt echo handling */ priv->echo_skb[idx] = skb; } else { diff --git a/drivers/net/can/flexcan.c b/drivers/net/can/flexcan.c index 7ab20a6b0d1d..202d08f8e1a4 100644 --- a/drivers/net/can/flexcan.c +++ b/drivers/net/can/flexcan.c @@ -815,7 +815,7 @@ static netdev_tx_t flexcan_start_xmit(struct sk_buff *skb, struct net_device *de priv->write(data, &priv->tx_mb->data[i / sizeof(u32)]); } - can_put_echo_skb(skb, dev, 0); + can_put_echo_skb(skb, dev, 0, 0); priv->write(can_id, &priv->tx_mb->can_id); priv->write(ctrl, &priv->tx_mb->can_ctrl); diff --git a/drivers/net/can/grcan.c b/drivers/net/can/grcan.c index f5d94a692576..8086cdc10000 100644 --- a/drivers/net/can/grcan.c +++ b/drivers/net/can/grcan.c @@ -1448,7 +1448,7 @@ static netdev_tx_t grcan_start_xmit(struct sk_buff *skb, * taken. */ priv->txdlc[slotindex] = cf->len; /* Store dlc for statistics */ - can_put_echo_skb(skb, dev, slotindex); + can_put_echo_skb(skb, dev, slotindex, 0); /* Make sure everything is written before allowing hardware to * read from the memory diff --git a/drivers/net/can/ifi_canfd/ifi_canfd.c b/drivers/net/can/ifi_canfd/ifi_canfd.c index 86b0e1406a21..56ac9e1dace7 100644 --- a/drivers/net/can/ifi_canfd/ifi_canfd.c +++ b/drivers/net/can/ifi_canfd/ifi_canfd.c @@ -922,7 +922,7 @@ static netdev_tx_t ifi_canfd_start_xmit(struct sk_buff *skb, writel(0, priv->base + IFI_CANFD_TXFIFO_REPEATCOUNT); writel(0, priv->base + IFI_CANFD_TXFIFO_SUSPEND_US); - can_put_echo_skb(skb, ndev, 0); + can_put_echo_skb(skb, ndev, 0, 0); /* Start the transmission */ writel(IFI_CANFD_TXSTCMD_ADD_MSG, priv->base + IFI_CANFD_TXSTCMD); diff --git a/drivers/net/can/kvaser_pciefd.c b/drivers/net/can/kvaser_pciefd.c index 969cedb9b0b6..0cf82f0646a3 100644 --- a/drivers/net/can/kvaser_pciefd.c +++ b/drivers/net/can/kvaser_pciefd.c @@ -778,7 +778,7 @@ static netdev_tx_t kvaser_pciefd_start_xmit(struct sk_buff *skb, spin_lock_irqsave(&can->echo_lock, irq_flags); /* Prepare and save echo skb in internal slot */ - can_put_echo_skb(skb, netdev, can->echo_idx); + can_put_echo_skb(skb, netdev, can->echo_idx, 0); /* Move echo index to the next slot */ can->echo_idx = (can->echo_idx + 1) % can->can.echo_skb_max; diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c index da551fd0f502..fff7432103cb 100644 --- a/drivers/net/can/m_can/m_can.c +++ b/drivers/net/can/m_can/m_can.c @@ -1483,7 +1483,7 @@ static netdev_tx_t m_can_tx_handler(struct m_can_classdev *cdev) M_CAN_FIFO_DATA(i / 4), *(u32 *)(cf->data + i)); - can_put_echo_skb(skb, dev, 0); + can_put_echo_skb(skb, dev, 0, 0); if (cdev->can.ctrlmode & CAN_CTRLMODE_FD) { cccr = m_can_read(cdev, M_CAN_CCCR); @@ -1554,7 +1554,7 @@ static netdev_tx_t m_can_tx_handler(struct m_can_classdev *cdev) /* Push loopback echo. * Will be looped back on TX interrupt based on message marker */ - can_put_echo_skb(skb, dev, putidx); + can_put_echo_skb(skb, dev, putidx, 0); /* Enable TX FIFO element to start transfer */ m_can_write(cdev, M_CAN_TXBAR, (1 << putidx)); diff --git a/drivers/net/can/mscan/mscan.c b/drivers/net/can/mscan/mscan.c index 5ed00a1558e1..a28fdaa411c6 100644 --- a/drivers/net/can/mscan/mscan.c +++ b/drivers/net/can/mscan/mscan.c @@ -270,7 +270,7 @@ static netdev_tx_t mscan_start_xmit(struct sk_buff *skb, struct net_device *dev) list_add_tail(&priv->tx_queue[buf_id].list, &priv->tx_head); - can_put_echo_skb(skb, dev, buf_id); + can_put_echo_skb(skb, dev, buf_id, 0); /* Enable interrupt. */ priv->tx_active |= 1 << buf_id; diff --git a/drivers/net/can/pch_can.c b/drivers/net/can/pch_can.c index 4f9e7ec192aa..a4c35b48d8e9 100644 --- a/drivers/net/can/pch_can.c +++ b/drivers/net/can/pch_can.c @@ -924,7 +924,7 @@ static netdev_tx_t pch_xmit(struct sk_buff *skb, struct net_device *ndev) &priv->regs->ifregs[1].data[i / 2]); } - can_put_echo_skb(skb, ndev, tx_obj_no - PCH_RX_OBJ_END - 1); + can_put_echo_skb(skb, ndev, tx_obj_no - PCH_RX_OBJ_END - 1, 0); /* Set the size of the data. Update if2_mcont */ iowrite32(cf->len | PCH_IF_MCONT_NEWDAT | PCH_IF_MCONT_TXRQXT | diff --git a/drivers/net/can/peak_canfd/peak_canfd.c b/drivers/net/can/peak_canfd/peak_canfd.c index c5334b0c3038..179a8e10fbb8 100644 --- a/drivers/net/can/peak_canfd/peak_canfd.c +++ b/drivers/net/can/peak_canfd/peak_canfd.c @@ -716,7 +716,7 @@ static netdev_tx_t peak_canfd_start_xmit(struct sk_buff *skb, spin_lock_irqsave(&priv->echo_lock, flags); /* prepare and save echo skb in internal slot */ - can_put_echo_skb(skb, ndev, priv->echo_idx); + can_put_echo_skb(skb, ndev, priv->echo_idx, 0); /* move echo index to the next slot */ priv->echo_idx = (priv->echo_idx + 1) % priv->can.echo_skb_max; diff --git a/drivers/net/can/rcar/rcar_can.c b/drivers/net/can/rcar/rcar_can.c index c803327f8f79..0b7e488bc4fe 100644 --- a/drivers/net/can/rcar/rcar_can.c +++ b/drivers/net/can/rcar/rcar_can.c @@ -617,7 +617,7 @@ static netdev_tx_t rcar_can_start_xmit(struct sk_buff *skb, writeb(cf->len, &priv->regs->mb[RCAR_CAN_TX_FIFO_MBX].dlc); priv->tx_dlc[priv->tx_head % RCAR_CAN_FIFO_DEPTH] = cf->len; - can_put_echo_skb(skb, ndev, priv->tx_head % RCAR_CAN_FIFO_DEPTH); + can_put_echo_skb(skb, ndev, priv->tx_head % RCAR_CAN_FIFO_DEPTH, 0); priv->tx_head++; /* Start Tx: write 0xff to the TFPCR register to increment * the CPU-side pointer for the transmit FIFO to the next diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index 2778ed5c61d1..38376f29bc56 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -1390,7 +1390,7 @@ static netdev_tx_t rcar_canfd_start_xmit(struct sk_buff *skb, } priv->tx_len[priv->tx_head % RCANFD_FIFO_DEPTH] = cf->len; - can_put_echo_skb(skb, ndev, priv->tx_head % RCANFD_FIFO_DEPTH); + can_put_echo_skb(skb, ndev, priv->tx_head % RCANFD_FIFO_DEPTH, 0); spin_lock_irqsave(&priv->tx_lock, flags); priv->tx_head++; diff --git a/drivers/net/can/sja1000/sja1000.c b/drivers/net/can/sja1000/sja1000.c index b6a7003c51d2..e98482c7bf33 100644 --- a/drivers/net/can/sja1000/sja1000.c +++ b/drivers/net/can/sja1000/sja1000.c @@ -318,7 +318,7 @@ static netdev_tx_t sja1000_start_xmit(struct sk_buff *skb, for (i = 0; i < cf->len; i++) priv->write_reg(priv, dreg++, cf->data[i]); - can_put_echo_skb(skb, dev, 0); + can_put_echo_skb(skb, dev, 0, 0); if (priv->can.ctrlmode & CAN_CTRLMODE_ONE_SHOT) cmd_reg_val |= CMD_AT; diff --git a/drivers/net/can/softing/softing_main.c b/drivers/net/can/softing/softing_main.c index 40070c930202..a5314448c5ae 100644 --- a/drivers/net/can/softing/softing_main.c +++ b/drivers/net/can/softing/softing_main.c @@ -104,7 +104,7 @@ static netdev_tx_t softing_netdev_start_xmit(struct sk_buff *skb, card->tx.last_bus = priv->index; ++card->tx.pending; ++priv->tx.pending; - can_put_echo_skb(skb, dev, priv->tx.echo_put); + can_put_echo_skb(skb, dev, priv->tx.echo_put, 0); ++priv->tx.echo_put; if (priv->tx.echo_put >= TX_ECHO_SKB_MAX) priv->tx.echo_put = 0; diff --git a/drivers/net/can/spi/hi311x.c b/drivers/net/can/spi/hi311x.c index f9455de94786..8c83a9e5a9e4 100644 --- a/drivers/net/can/spi/hi311x.c +++ b/drivers/net/can/spi/hi311x.c @@ -586,7 +586,7 @@ static void hi3110_tx_work_handler(struct work_struct *ws) frame = (struct can_frame *)priv->tx_skb->data; hi3110_hw_tx(spi, frame); priv->tx_len = 1 + frame->len; - can_put_echo_skb(priv->tx_skb, net, 0); + can_put_echo_skb(priv->tx_skb, net, 0, 0); priv->tx_skb = NULL; } } diff --git a/drivers/net/can/spi/mcp251x.c b/drivers/net/can/spi/mcp251x.c index 25859d16d06f..40866754aafc 100644 --- a/drivers/net/can/spi/mcp251x.c +++ b/drivers/net/can/spi/mcp251x.c @@ -1002,7 +1002,7 @@ static void mcp251x_tx_work_handler(struct work_struct *ws) frame->len = CAN_FRAME_MAX_DATA_LEN; mcp251x_hw_tx(spi, frame, 0); priv->tx_len = 1 + frame->len; - can_put_echo_skb(priv->tx_skb, net, 0); + can_put_echo_skb(priv->tx_skb, net, 0, 0); priv->tx_skb = NULL; } } diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c index 36235afb0bc6..95bba456a4cd 100644 --- a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c @@ -2436,7 +2436,7 @@ static netdev_tx_t mcp251xfd_start_xmit(struct sk_buff *skb, if (tx_ring->head - tx_ring->tail >= tx_ring->obj_num) netif_stop_queue(ndev); - can_put_echo_skb(skb, ndev, tx_head); + can_put_echo_skb(skb, ndev, tx_head, 0); err = mcp251xfd_tx_obj_write(priv, tx_obj); if (err) diff --git a/drivers/net/can/sun4i_can.c b/drivers/net/can/sun4i_can.c index 783b63218b7b..b75175d59104 100644 --- a/drivers/net/can/sun4i_can.c +++ b/drivers/net/can/sun4i_can.c @@ -448,7 +448,7 @@ static netdev_tx_t sun4ican_start_xmit(struct sk_buff *skb, struct net_device *d writel(msg_flag_n, priv->base + SUN4I_REG_BUF0_ADDR); - can_put_echo_skb(skb, dev, 0); + can_put_echo_skb(skb, dev, 0, 0); if (priv->can.ctrlmode & CAN_CTRLMODE_LOOPBACK) sun4i_can_write_cmdreg(priv, SUN4I_CMD_SELF_RCV_REQ); diff --git a/drivers/net/can/ti_hecc.c b/drivers/net/can/ti_hecc.c index a6850ff0b55b..485c19bc98c2 100644 --- a/drivers/net/can/ti_hecc.c +++ b/drivers/net/can/ti_hecc.c @@ -513,7 +513,7 @@ static netdev_tx_t ti_hecc_xmit(struct sk_buff *skb, struct net_device *ndev) be32_to_cpu(*(__be32 *)(cf->data + 4))); else *(u32 *)(cf->data + 4) = 0; - can_put_echo_skb(skb, ndev, mbxno); + can_put_echo_skb(skb, ndev, mbxno, 0); spin_lock_irqsave(&priv->mbx_lock, flags); --priv->tx_head; diff --git a/drivers/net/can/usb/ems_usb.c b/drivers/net/can/usb/ems_usb.c index 25eee4466364..5e5330060464 100644 --- a/drivers/net/can/usb/ems_usb.c +++ b/drivers/net/can/usb/ems_usb.c @@ -801,7 +801,7 @@ static netdev_tx_t ems_usb_start_xmit(struct sk_buff *skb, struct net_device *ne urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP; usb_anchor_urb(urb, &dev->tx_submitted); - can_put_echo_skb(skb, netdev, context->echo_index); + can_put_echo_skb(skb, netdev, context->echo_index, 0); atomic_inc(&dev->active_tx_urbs); diff --git a/drivers/net/can/usb/esd_usb2.c b/drivers/net/can/usb/esd_usb2.c index 9eed75a4b678..68d8a85f00c4 100644 --- a/drivers/net/can/usb/esd_usb2.c +++ b/drivers/net/can/usb/esd_usb2.c @@ -783,7 +783,7 @@ static netdev_tx_t esd_usb2_start_xmit(struct sk_buff *skb, usb_anchor_urb(urb, &priv->tx_submitted); - can_put_echo_skb(skb, netdev, context->echo_index); + can_put_echo_skb(skb, netdev, context->echo_index, 0); atomic_inc(&priv->active_tx_jobs); diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c index 0487095e1fd0..5ce9ba5d29d6 100644 --- a/drivers/net/can/usb/gs_usb.c +++ b/drivers/net/can/usb/gs_usb.c @@ -525,7 +525,7 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb, urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP; usb_anchor_urb(urb, &dev->tx_submitted); - can_put_echo_skb(skb, netdev, idx); + can_put_echo_skb(skb, netdev, idx, 0); atomic_inc(&dev->active_tx_urbs); diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c index e2d58846c40c..2b7efd296758 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c @@ -578,7 +578,7 @@ static netdev_tx_t kvaser_usb_start_xmit(struct sk_buff *skb, context->priv = priv; - can_put_echo_skb(skb, netdev, context->echo_index); + can_put_echo_skb(skb, netdev, context->echo_index, 0); usb_fill_bulk_urb(urb, dev->udev, usb_sndbulkpipe(dev->udev, diff --git a/drivers/net/can/usb/mcba_usb.c b/drivers/net/can/usb/mcba_usb.c index df54eb7d4b36..5347c89992ce 100644 --- a/drivers/net/can/usb/mcba_usb.c +++ b/drivers/net/can/usb/mcba_usb.c @@ -355,7 +355,7 @@ static netdev_tx_t mcba_usb_start_xmit(struct sk_buff *skb, if (cf->can_id & CAN_RTR_FLAG) usb_msg.dlc |= MCBA_DLC_RTR_MASK; - can_put_echo_skb(skb, priv->netdev, ctx->ndx); + can_put_echo_skb(skb, priv->netdev, ctx->ndx, 0); err = mcba_usb_xmit(priv, (struct mcba_usb_msg *)&usb_msg, ctx); if (err) diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c index 251835ea15aa..95672750419a 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c @@ -365,7 +365,7 @@ static netdev_tx_t peak_usb_ndo_start_xmit(struct sk_buff *skb, usb_anchor_urb(urb, &dev->tx_submitted); - can_put_echo_skb(skb, netdev, context->echo_index); + can_put_echo_skb(skb, netdev, context->echo_index, 0); atomic_inc(&dev->active_tx_urbs); diff --git a/drivers/net/can/usb/ucan.c b/drivers/net/can/usb/ucan.c index 7d92da8954fe..5add27614e2b 100644 --- a/drivers/net/can/usb/ucan.c +++ b/drivers/net/can/usb/ucan.c @@ -1137,7 +1137,7 @@ static netdev_tx_t ucan_start_xmit(struct sk_buff *skb, /* put the skb on can loopback stack */ spin_lock_irqsave(&up->echo_skb_lock, flags); - can_put_echo_skb(skb, up->netdev, echo_index); + can_put_echo_skb(skb, up->netdev, echo_index, 0); spin_unlock_irqrestore(&up->echo_skb_lock, flags); /* transmit it */ diff --git a/drivers/net/can/usb/usb_8dev.c b/drivers/net/can/usb/usb_8dev.c index 44478304ff46..2e824d9d8167 100644 --- a/drivers/net/can/usb/usb_8dev.c +++ b/drivers/net/can/usb/usb_8dev.c @@ -664,7 +664,7 @@ static netdev_tx_t usb_8dev_start_xmit(struct sk_buff *skb, urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP; usb_anchor_urb(urb, &priv->tx_submitted); - can_put_echo_skb(skb, netdev, context->echo_index); + can_put_echo_skb(skb, netdev, context->echo_index, 0); atomic_inc(&priv->active_tx_urbs); diff --git a/drivers/net/can/xilinx_can.c b/drivers/net/can/xilinx_can.c index 3f54edee92eb..8d5132a3f2c9 100644 --- a/drivers/net/can/xilinx_can.c +++ b/drivers/net/can/xilinx_can.c @@ -592,9 +592,9 @@ static void xcan_write_frame(struct net_device *ndev, struct sk_buff *skb, if (!(priv->devtype.flags & XCAN_FLAG_TX_MAILBOXES) && (priv->devtype.flags & XCAN_FLAG_TXFEMP)) - can_put_echo_skb(skb, ndev, priv->tx_head % priv->tx_max); + can_put_echo_skb(skb, ndev, priv->tx_head % priv->tx_max, 0); else - can_put_echo_skb(skb, ndev, 0); + can_put_echo_skb(skb, ndev, 0, 0); priv->tx_head++; diff --git a/include/linux/can/skb.h b/include/linux/can/skb.h index 5db9da30843c..eaac4a637ae0 100644 --- a/include/linux/can/skb.h +++ b/include/linux/can/skb.h @@ -18,7 +18,7 @@ void can_flush_echo_skb(struct net_device *dev); int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, - unsigned int idx); + unsigned int idx, unsigned int frame_len); struct sk_buff *__can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr, unsigned int *frame_len_ptr); unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx); -- cgit v1.2.3 From 9420e1d495e2a3b5f673148b7e3ebc861b1441f7 Mon Sep 17 00:00:00 2001 From: Marc Kleine-Budde Date: Mon, 11 Jan 2021 15:19:28 +0100 Subject: can: dev: can_get_echo_skb(): extend to return can frame length In order to implement byte queue limits (bql) in CAN drivers, the length of the CAN frame needs to be passed into the networking stack after queueing and after transmission completion. To avoid to calculate this length twice, extend can_get_echo_skb() to return that value. Convert all users of this function, too. Reviewed-by: Vincent Mailhol Link: https://lore.kernel.org/r/20210111141930.693847-14-mkl@pengutronix.de Signed-off-by: Marc Kleine-Budde --- drivers/net/can/at91_can.c | 2 +- drivers/net/can/c_can/c_can.c | 2 +- drivers/net/can/cc770/cc770.c | 2 +- drivers/net/can/dev/skb.c | 5 +++-- drivers/net/can/grcan.c | 2 +- drivers/net/can/ifi_canfd/ifi_canfd.c | 2 +- drivers/net/can/kvaser_pciefd.c | 4 ++-- drivers/net/can/m_can/m_can.c | 4 ++-- drivers/net/can/mscan/mscan.c | 2 +- drivers/net/can/pch_can.c | 2 +- drivers/net/can/peak_canfd/peak_canfd.c | 2 +- drivers/net/can/rcar/rcar_can.c | 2 +- drivers/net/can/rcar/rcar_canfd.c | 2 +- drivers/net/can/sja1000/sja1000.c | 2 +- drivers/net/can/softing/softing_main.c | 2 +- drivers/net/can/spi/hi311x.c | 2 +- drivers/net/can/spi/mcp251x.c | 2 +- drivers/net/can/sun4i_can.c | 2 +- drivers/net/can/usb/ems_usb.c | 2 +- drivers/net/can/usb/esd_usb2.c | 2 +- drivers/net/can/usb/gs_usb.c | 2 +- drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c | 2 +- drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c | 2 +- drivers/net/can/usb/mcba_usb.c | 2 +- drivers/net/can/usb/peak_usb/pcan_usb_core.c | 2 +- drivers/net/can/usb/ucan.c | 2 +- drivers/net/can/usb/usb_8dev.c | 2 +- drivers/net/can/xilinx_can.c | 2 +- include/linux/can/skb.h | 3 ++- 29 files changed, 34 insertions(+), 32 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/can/at91_can.c b/drivers/net/can/at91_can.c index 90b223a80ed4..9ad9b39f480e 100644 --- a/drivers/net/can/at91_can.c +++ b/drivers/net/can/at91_can.c @@ -856,7 +856,7 @@ static void at91_irq_tx(struct net_device *dev, u32 reg_sr) if (likely(reg_msr & AT91_MSR_MRDY && ~reg_msr & AT91_MSR_MABT)) { /* _NOTE_: subtract AT91_MB_TX_FIRST offset from mb! */ - can_get_echo_skb(dev, mb - get_mb_tx_first(priv)); + can_get_echo_skb(dev, mb - get_mb_tx_first(priv), NULL); dev->stats.tx_packets++; can_led_event(dev, CAN_LED_EVENT_TX); } diff --git a/drivers/net/can/c_can/c_can.c b/drivers/net/can/c_can/c_can.c index 13638954a25c..ef474bae47a1 100644 --- a/drivers/net/can/c_can/c_can.c +++ b/drivers/net/can/c_can/c_can.c @@ -733,7 +733,7 @@ static void c_can_do_tx(struct net_device *dev) pend &= ~(1 << idx); obj = idx + C_CAN_MSG_OBJ_TX_FIRST; c_can_inval_tx_object(dev, IF_RX, obj); - can_get_echo_skb(dev, idx); + can_get_echo_skb(dev, idx, NULL); bytes += priv->dlc[idx]; pkts++; } diff --git a/drivers/net/can/cc770/cc770.c b/drivers/net/can/cc770/cc770.c index e53ca338368a..f8a130f594e2 100644 --- a/drivers/net/can/cc770/cc770.c +++ b/drivers/net/can/cc770/cc770.c @@ -703,7 +703,7 @@ static void cc770_tx_interrupt(struct net_device *dev, unsigned int o) stats->tx_packets++; can_put_echo_skb(priv->tx_skb, dev, 0, 0); - can_get_echo_skb(dev, 0); + can_get_echo_skb(dev, 0, NULL); priv->tx_skb = NULL; netif_wake_queue(dev); diff --git a/drivers/net/can/dev/skb.c b/drivers/net/can/dev/skb.c index c184b4dce19e..53683d4312f1 100644 --- a/drivers/net/can/dev/skb.c +++ b/drivers/net/can/dev/skb.c @@ -121,12 +121,13 @@ __can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr, * is handled in the device driver. The driver must protect * access to priv->echo_skb, if necessary. */ -unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx) +unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx, + unsigned int *frame_len_ptr) { struct sk_buff *skb; u8 len; - skb = __can_get_echo_skb(dev, idx, &len, NULL); + skb = __can_get_echo_skb(dev, idx, &len, frame_len_ptr); if (!skb) return 0; diff --git a/drivers/net/can/grcan.c b/drivers/net/can/grcan.c index 8086cdc10000..4a8453290530 100644 --- a/drivers/net/can/grcan.c +++ b/drivers/net/can/grcan.c @@ -517,7 +517,7 @@ static int catch_up_echo_skb(struct net_device *dev, int budget, bool echo) stats->tx_packets++; stats->tx_bytes += priv->txdlc[i]; priv->txdlc[i] = 0; - can_get_echo_skb(dev, i); + can_get_echo_skb(dev, i, NULL); } else { /* For cleanup of untransmitted messages */ can_free_echo_skb(dev, i); diff --git a/drivers/net/can/ifi_canfd/ifi_canfd.c b/drivers/net/can/ifi_canfd/ifi_canfd.c index 56ac9e1dace7..5bb957a26bc6 100644 --- a/drivers/net/can/ifi_canfd/ifi_canfd.c +++ b/drivers/net/can/ifi_canfd/ifi_canfd.c @@ -629,7 +629,7 @@ static irqreturn_t ifi_canfd_isr(int irq, void *dev_id) /* TX IRQ */ if (isr & IFI_CANFD_INTERRUPT_TXFIFO_REMOVE) { - stats->tx_bytes += can_get_echo_skb(ndev, 0); + stats->tx_bytes += can_get_echo_skb(ndev, 0, NULL); stats->tx_packets++; can_led_event(ndev, CAN_LED_EVENT_TX); } diff --git a/drivers/net/can/kvaser_pciefd.c b/drivers/net/can/kvaser_pciefd.c index 0cf82f0646a3..37e05010ca91 100644 --- a/drivers/net/can/kvaser_pciefd.c +++ b/drivers/net/can/kvaser_pciefd.c @@ -1467,7 +1467,7 @@ static int kvaser_pciefd_handle_eack_packet(struct kvaser_pciefd *pcie, can->reg_base + KVASER_PCIEFD_KCAN_CTRL_REG); } else { int echo_idx = p->header[0] & KVASER_PCIEFD_PACKET_SEQ_MSK; - int dlc = can_get_echo_skb(can->can.dev, echo_idx); + int dlc = can_get_echo_skb(can->can.dev, echo_idx, NULL); struct net_device_stats *stats = &can->can.dev->stats; stats->tx_bytes += dlc; @@ -1533,7 +1533,7 @@ static int kvaser_pciefd_handle_ack_packet(struct kvaser_pciefd *pcie, netdev_dbg(can->can.dev, "Packet was flushed\n"); } else { int echo_idx = p->header[0] & KVASER_PCIEFD_PACKET_SEQ_MSK; - int dlc = can_get_echo_skb(can->can.dev, echo_idx); + int dlc = can_get_echo_skb(can->can.dev, echo_idx, NULL); u8 count = ioread32(can->reg_base + KVASER_PCIEFD_KCAN_TX_NPACKETS_REG) & 0xff; diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c index fff7432103cb..3752520a7d4b 100644 --- a/drivers/net/can/m_can/m_can.c +++ b/drivers/net/can/m_can/m_can.c @@ -930,7 +930,7 @@ static void m_can_echo_tx_event(struct net_device *dev) (fgi << TXEFA_EFAI_SHIFT))); /* update stats */ - stats->tx_bytes += can_get_echo_skb(dev, msg_mark); + stats->tx_bytes += can_get_echo_skb(dev, msg_mark, NULL); stats->tx_packets++; } } @@ -972,7 +972,7 @@ static irqreturn_t m_can_isr(int irq, void *dev_id) if (cdev->version == 30) { if (ir & IR_TC) { /* Transmission Complete Interrupt*/ - stats->tx_bytes += can_get_echo_skb(dev, 0); + stats->tx_bytes += can_get_echo_skb(dev, 0, NULL); stats->tx_packets++; can_led_event(dev, CAN_LED_EVENT_TX); netif_wake_queue(dev); diff --git a/drivers/net/can/mscan/mscan.c b/drivers/net/can/mscan/mscan.c index a28fdaa411c6..fa32e418eb29 100644 --- a/drivers/net/can/mscan/mscan.c +++ b/drivers/net/can/mscan/mscan.c @@ -448,7 +448,7 @@ static irqreturn_t mscan_isr(int irq, void *dev_id) out_8(®s->cantbsel, mask); stats->tx_bytes += in_8(®s->tx.dlr); stats->tx_packets++; - can_get_echo_skb(dev, entry->id); + can_get_echo_skb(dev, entry->id, NULL); priv->tx_active &= ~mask; list_del(pos); } diff --git a/drivers/net/can/pch_can.c b/drivers/net/can/pch_can.c index a4c35b48d8e9..92a54a5fd4c5 100644 --- a/drivers/net/can/pch_can.c +++ b/drivers/net/can/pch_can.c @@ -711,7 +711,7 @@ static void pch_can_tx_complete(struct net_device *ndev, u32 int_stat) struct net_device_stats *stats = &(priv->ndev->stats); u32 dlc; - can_get_echo_skb(ndev, int_stat - PCH_RX_OBJ_END - 1); + can_get_echo_skb(ndev, int_stat - PCH_RX_OBJ_END - 1, NULL); iowrite32(PCH_CMASK_RX_TX_GET | PCH_CMASK_CLRINTPND, &priv->regs->ifregs[1].cmask); pch_can_rw_msg_obj(&priv->regs->ifregs[1].creq, int_stat); diff --git a/drivers/net/can/peak_canfd/peak_canfd.c b/drivers/net/can/peak_canfd/peak_canfd.c index 179a8e10fbb8..00847cbaf7b6 100644 --- a/drivers/net/can/peak_canfd/peak_canfd.c +++ b/drivers/net/can/peak_canfd/peak_canfd.c @@ -266,7 +266,7 @@ static int pucan_handle_can_rx(struct peak_canfd_priv *priv, unsigned long flags; spin_lock_irqsave(&priv->echo_lock, flags); - can_get_echo_skb(priv->ndev, msg->client); + can_get_echo_skb(priv->ndev, msg->client, NULL); /* count bytes of the echo instead of skb */ stats->tx_bytes += cf_len; diff --git a/drivers/net/can/rcar/rcar_can.c b/drivers/net/can/rcar/rcar_can.c index 0b7e488bc4fe..4870c4ea190a 100644 --- a/drivers/net/can/rcar/rcar_can.c +++ b/drivers/net/can/rcar/rcar_can.c @@ -386,7 +386,7 @@ static void rcar_can_tx_done(struct net_device *ndev) stats->tx_bytes += priv->tx_dlc[priv->tx_tail % RCAR_CAN_FIFO_DEPTH]; priv->tx_dlc[priv->tx_tail % RCAR_CAN_FIFO_DEPTH] = 0; - can_get_echo_skb(ndev, priv->tx_tail % RCAR_CAN_FIFO_DEPTH); + can_get_echo_skb(ndev, priv->tx_tail % RCAR_CAN_FIFO_DEPTH, NULL); priv->tx_tail++; netif_wake_queue(ndev); } diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index 38376f29bc56..d8d233e62990 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -1044,7 +1044,7 @@ static void rcar_canfd_tx_done(struct net_device *ndev) stats->tx_packets++; stats->tx_bytes += priv->tx_len[sent]; priv->tx_len[sent] = 0; - can_get_echo_skb(ndev, sent); + can_get_echo_skb(ndev, sent, NULL); spin_lock_irqsave(&priv->tx_lock, flags); priv->tx_tail++; diff --git a/drivers/net/can/sja1000/sja1000.c b/drivers/net/can/sja1000/sja1000.c index e98482c7bf33..9e86488ba55f 100644 --- a/drivers/net/can/sja1000/sja1000.c +++ b/drivers/net/can/sja1000/sja1000.c @@ -531,7 +531,7 @@ irqreturn_t sja1000_interrupt(int irq, void *dev_id) stats->tx_bytes += priv->read_reg(priv, SJA1000_FI) & 0xf; stats->tx_packets++; - can_get_echo_skb(dev, 0); + can_get_echo_skb(dev, 0, NULL); } netif_wake_queue(dev); can_led_event(dev, CAN_LED_EVENT_TX); diff --git a/drivers/net/can/softing/softing_main.c b/drivers/net/can/softing/softing_main.c index a5314448c5ae..c44f3411e561 100644 --- a/drivers/net/can/softing/softing_main.c +++ b/drivers/net/can/softing/softing_main.c @@ -284,7 +284,7 @@ static int softing_handle_1(struct softing *card) skb = priv->can.echo_skb[priv->tx.echo_get]; if (skb) skb->tstamp = ktime; - can_get_echo_skb(netdev, priv->tx.echo_get); + can_get_echo_skb(netdev, priv->tx.echo_get, NULL); ++priv->tx.echo_get; if (priv->tx.echo_get >= TX_ECHO_SKB_MAX) priv->tx.echo_get = 0; diff --git a/drivers/net/can/spi/hi311x.c b/drivers/net/can/spi/hi311x.c index 8c83a9e5a9e4..c3e020c90111 100644 --- a/drivers/net/can/spi/hi311x.c +++ b/drivers/net/can/spi/hi311x.c @@ -725,7 +725,7 @@ static irqreturn_t hi3110_can_ist(int irq, void *dev_id) net->stats.tx_bytes += priv->tx_len - 1; can_led_event(net, CAN_LED_EVENT_TX); if (priv->tx_len) { - can_get_echo_skb(net, 0); + can_get_echo_skb(net, 0, NULL); priv->tx_len = 0; } netif_wake_queue(net); diff --git a/drivers/net/can/spi/mcp251x.c b/drivers/net/can/spi/mcp251x.c index 40866754aafc..f69fb4238a65 100644 --- a/drivers/net/can/spi/mcp251x.c +++ b/drivers/net/can/spi/mcp251x.c @@ -1171,7 +1171,7 @@ static irqreturn_t mcp251x_can_ist(int irq, void *dev_id) net->stats.tx_bytes += priv->tx_len - 1; can_led_event(net, CAN_LED_EVENT_TX); if (priv->tx_len) { - can_get_echo_skb(net, 0); + can_get_echo_skb(net, 0, NULL); priv->tx_len = 0; } netif_wake_queue(net); diff --git a/drivers/net/can/sun4i_can.c b/drivers/net/can/sun4i_can.c index b75175d59104..54aa7c25c4de 100644 --- a/drivers/net/can/sun4i_can.c +++ b/drivers/net/can/sun4i_can.c @@ -655,7 +655,7 @@ static irqreturn_t sun4i_can_interrupt(int irq, void *dev_id) readl(priv->base + SUN4I_REG_RBUF_RBACK_START_ADDR) & 0xf; stats->tx_packets++; - can_get_echo_skb(dev, 0); + can_get_echo_skb(dev, 0, NULL); netif_wake_queue(dev); can_led_event(dev, CAN_LED_EVENT_TX); } diff --git a/drivers/net/can/usb/ems_usb.c b/drivers/net/can/usb/ems_usb.c index 5e5330060464..18f40eb20360 100644 --- a/drivers/net/can/usb/ems_usb.c +++ b/drivers/net/can/usb/ems_usb.c @@ -518,7 +518,7 @@ static void ems_usb_write_bulk_callback(struct urb *urb) netdev->stats.tx_packets++; netdev->stats.tx_bytes += context->dlc; - can_get_echo_skb(netdev, context->echo_index); + can_get_echo_skb(netdev, context->echo_index, NULL); /* Release context */ context->echo_index = MAX_TX_URBS; diff --git a/drivers/net/can/usb/esd_usb2.c b/drivers/net/can/usb/esd_usb2.c index 68d8a85f00c4..562acbf454fd 100644 --- a/drivers/net/can/usb/esd_usb2.c +++ b/drivers/net/can/usb/esd_usb2.c @@ -357,7 +357,7 @@ static void esd_usb2_tx_done_msg(struct esd_usb2_net_priv *priv, if (!msg->msg.txdone.status) { stats->tx_packets++; stats->tx_bytes += context->len; - can_get_echo_skb(netdev, context->echo_index); + can_get_echo_skb(netdev, context->echo_index, NULL); } else { stats->tx_errors++; can_free_echo_skb(netdev, context->echo_index); diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c index 5ce9ba5d29d6..a00dc1904415 100644 --- a/drivers/net/can/usb/gs_usb.c +++ b/drivers/net/can/usb/gs_usb.c @@ -370,7 +370,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) goto resubmit_urb; } - can_get_echo_skb(netdev, hf->echo_id); + can_get_echo_skb(netdev, hf->echo_id, NULL); gs_free_tx_context(txc); diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c index 480bd2ecb296..dcee8dc828ec 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c @@ -1151,7 +1151,7 @@ static void kvaser_usb_hydra_tx_acknowledge(const struct kvaser_usb *dev, spin_lock_irqsave(&priv->tx_contexts_lock, irq_flags); - can_get_echo_skb(priv->netdev, context->echo_index); + can_get_echo_skb(priv->netdev, context->echo_index, NULL); context->echo_index = dev->max_tx_urbs; --priv->active_tx_contexts; netif_wake_queue(priv->netdev); diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c index 98c016ef0607..59ba7c7beec0 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c @@ -594,7 +594,7 @@ static void kvaser_usb_leaf_tx_acknowledge(const struct kvaser_usb *dev, spin_lock_irqsave(&priv->tx_contexts_lock, flags); - can_get_echo_skb(priv->netdev, context->echo_index); + can_get_echo_skb(priv->netdev, context->echo_index, NULL); context->echo_index = dev->max_tx_urbs; --priv->active_tx_contexts; netif_wake_queue(priv->netdev); diff --git a/drivers/net/can/usb/mcba_usb.c b/drivers/net/can/usb/mcba_usb.c index 5347c89992ce..4232a7126c1b 100644 --- a/drivers/net/can/usb/mcba_usb.c +++ b/drivers/net/can/usb/mcba_usb.c @@ -237,7 +237,7 @@ static void mcba_usb_write_bulk_callback(struct urb *urb) netdev->stats.tx_bytes += ctx->dlc; can_led_event(netdev, CAN_LED_EVENT_TX); - can_get_echo_skb(netdev, ctx->ndx); + can_get_echo_skb(netdev, ctx->ndx, NULL); } if (urb->status) diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c index 95672750419a..573b11559d73 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c @@ -309,7 +309,7 @@ static void peak_usb_write_bulk_callback(struct urb *urb) } /* should always release echo skb and corresponding context */ - can_get_echo_skb(netdev, context->echo_index); + can_get_echo_skb(netdev, context->echo_index, NULL); context->echo_index = PCAN_USB_MAX_TX_URBS; /* do wakeup tx queue in case of success only */ diff --git a/drivers/net/can/usb/ucan.c b/drivers/net/can/usb/ucan.c index 5add27614e2b..fa403c080871 100644 --- a/drivers/net/can/usb/ucan.c +++ b/drivers/net/can/usb/ucan.c @@ -672,7 +672,7 @@ static void ucan_tx_complete_msg(struct ucan_priv *up, /* update statistics */ up->netdev->stats.tx_packets++; up->netdev->stats.tx_bytes += dlc; - can_get_echo_skb(up->netdev, echo_index); + can_get_echo_skb(up->netdev, echo_index, NULL); } else { up->netdev->stats.tx_dropped++; can_free_echo_skb(up->netdev, echo_index); diff --git a/drivers/net/can/usb/usb_8dev.c b/drivers/net/can/usb/usb_8dev.c index 2e824d9d8167..e8c42430a4fc 100644 --- a/drivers/net/can/usb/usb_8dev.c +++ b/drivers/net/can/usb/usb_8dev.c @@ -585,7 +585,7 @@ static void usb_8dev_write_bulk_callback(struct urb *urb) netdev->stats.tx_packets++; netdev->stats.tx_bytes += context->dlc; - can_get_echo_skb(netdev, context->echo_index); + can_get_echo_skb(netdev, context->echo_index, NULL); can_led_event(netdev, CAN_LED_EVENT_TX); diff --git a/drivers/net/can/xilinx_can.c b/drivers/net/can/xilinx_can.c index 8d5132a3f2c9..37fa19c62d73 100644 --- a/drivers/net/can/xilinx_can.c +++ b/drivers/net/can/xilinx_can.c @@ -1292,7 +1292,7 @@ static void xcan_tx_interrupt(struct net_device *ndev, u32 isr) while (frames_sent--) { stats->tx_bytes += can_get_echo_skb(ndev, priv->tx_tail % - priv->tx_max); + priv->tx_max, NULL); priv->tx_tail++; stats->tx_packets++; } diff --git a/include/linux/can/skb.h b/include/linux/can/skb.h index eaac4a637ae0..685f34cfba20 100644 --- a/include/linux/can/skb.h +++ b/include/linux/can/skb.h @@ -21,7 +21,8 @@ int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, unsigned int idx, unsigned int frame_len); struct sk_buff *__can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr, unsigned int *frame_len_ptr); -unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx); +unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx, + unsigned int *frame_len_ptr); void can_free_echo_skb(struct net_device *dev, unsigned int idx); struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf); struct sk_buff *alloc_canfd_skb(struct net_device *dev, -- cgit v1.2.3 From 99842c9685ab00fa8689e8bd12bde62b706b6198 Mon Sep 17 00:00:00 2001 From: Marc Kleine-Budde Date: Mon, 11 Jan 2021 15:19:29 +0100 Subject: can: dev: can_rx_offload_get_echo_skb(): extend to return can frame length In order to implement byte queue limits (bql) in CAN drivers, the length of the CAN frame needs to be passed into the networking stack after queueing and after transmission completion. To avoid to calculate this length twice, extend can_rx_offload_get_echo_skb() to return that value. Convert all users of this function, too. Reviewed-by: Vincent Mailhol Link: https://lore.kernel.org/r/20210111141930.693847-15-mkl@pengutronix.de Signed-off-by: Marc Kleine-Budde --- drivers/net/can/dev/rx-offload.c | 5 +++-- drivers/net/can/flexcan.c | 5 +++-- drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c | 2 +- drivers/net/can/ti_hecc.c | 2 +- include/linux/can/rx-offload.h | 3 ++- 5 files changed, 10 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/can/dev/rx-offload.c b/drivers/net/can/dev/rx-offload.c index 6a26b5282df1..ab2c1543786c 100644 --- a/drivers/net/can/dev/rx-offload.c +++ b/drivers/net/can/dev/rx-offload.c @@ -263,7 +263,8 @@ int can_rx_offload_queue_sorted(struct can_rx_offload *offload, EXPORT_SYMBOL_GPL(can_rx_offload_queue_sorted); unsigned int can_rx_offload_get_echo_skb(struct can_rx_offload *offload, - unsigned int idx, u32 timestamp) + unsigned int idx, u32 timestamp, + unsigned int *frame_len_ptr) { struct net_device *dev = offload->dev; struct net_device_stats *stats = &dev->stats; @@ -271,7 +272,7 @@ unsigned int can_rx_offload_get_echo_skb(struct can_rx_offload *offload, u8 len; int err; - skb = __can_get_echo_skb(dev, idx, &len, NULL); + skb = __can_get_echo_skb(dev, idx, &len, frame_len_ptr); if (!skb) return 0; diff --git a/drivers/net/can/flexcan.c b/drivers/net/can/flexcan.c index 202d08f8e1a4..5d9157c655e9 100644 --- a/drivers/net/can/flexcan.c +++ b/drivers/net/can/flexcan.c @@ -1122,8 +1122,9 @@ static irqreturn_t flexcan_irq(int irq, void *dev_id) u32 reg_ctrl = priv->read(&priv->tx_mb->can_ctrl); handled = IRQ_HANDLED; - stats->tx_bytes += can_rx_offload_get_echo_skb(&priv->offload, - 0, reg_ctrl << 16); + stats->tx_bytes += + can_rx_offload_get_echo_skb(&priv->offload, 0, + reg_ctrl << 16, NULL); stats->tx_packets++; can_led_event(dev, CAN_LED_EVENT_TX); diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c index 95bba456a4cd..63bbe0930e53 100644 --- a/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-core.c @@ -1271,7 +1271,7 @@ mcp251xfd_handle_tefif_one(struct mcp251xfd_priv *priv, stats->tx_bytes += can_rx_offload_get_echo_skb(&priv->offload, mcp251xfd_get_tef_tail(priv), - hw_tef_obj->ts); + hw_tef_obj->ts, NULL); stats->tx_packets++; priv->tef->tail++; diff --git a/drivers/net/can/ti_hecc.c b/drivers/net/can/ti_hecc.c index 485c19bc98c2..73245d8836a9 100644 --- a/drivers/net/can/ti_hecc.c +++ b/drivers/net/can/ti_hecc.c @@ -757,7 +757,7 @@ static irqreturn_t ti_hecc_interrupt(int irq, void *dev_id) stamp = hecc_read_stamp(priv, mbxno); stats->tx_bytes += can_rx_offload_get_echo_skb(&priv->offload, - mbxno, stamp); + mbxno, stamp, NULL); stats->tx_packets++; can_led_event(ndev, CAN_LED_EVENT_TX); --priv->tx_tail; diff --git a/include/linux/can/rx-offload.h b/include/linux/can/rx-offload.h index f1b38088b765..40882df7105e 100644 --- a/include/linux/can/rx-offload.h +++ b/include/linux/can/rx-offload.h @@ -44,7 +44,8 @@ int can_rx_offload_irq_offload_fifo(struct can_rx_offload *offload); int can_rx_offload_queue_sorted(struct can_rx_offload *offload, struct sk_buff *skb, u32 timestamp); unsigned int can_rx_offload_get_echo_skb(struct can_rx_offload *offload, - unsigned int idx, u32 timestamp); + unsigned int idx, u32 timestamp, + unsigned int *frame_len_ptr); int can_rx_offload_queue_tail(struct can_rx_offload *offload, struct sk_buff *skb); void can_rx_offload_del(struct can_rx_offload *offload); -- cgit v1.2.3 From a5418be9dffe70ccbb0b4bd5ea3881c81927e965 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 8 Dec 2020 09:46:56 +0530 Subject: sched/core: Rename schedutil_cpu_util() and allow rest of the kernel to use it There is nothing schedutil specific in schedutil_cpu_util(), rename it to effective_cpu_util(). Also create and expose another wrapper sched_cpu_util() which can be used by other parts of the kernel, like thermal core (that will be done in a later commit). Signed-off-by: Viresh Kumar Signed-off-by: Peter Zijlstra (Intel) Acked-by: Rafael J. Wysocki Link: https://lkml.kernel.org/r/db011961fb3bb8bef1c0eda5cd64564637d3ef31.1607400596.git.viresh.kumar@linaro.org --- include/linux/sched.h | 5 +++++ kernel/sched/core.c | 10 ++++++++-- kernel/sched/cpufreq_schedutil.c | 2 +- kernel/sched/fair.c | 6 +++--- kernel/sched/sched.h | 10 +++++----- 5 files changed, 22 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index 6e3a5eeec509..31169e70476a 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1968,6 +1968,11 @@ extern long sched_getaffinity(pid_t pid, struct cpumask *mask); #define TASK_SIZE_OF(tsk) TASK_SIZE #endif +#ifdef CONFIG_SMP +/* Returns effective CPU energy utilization, as seen by the scheduler */ +unsigned long sched_cpu_util(int cpu, unsigned long max); +#endif /* CONFIG_SMP */ + #ifdef CONFIG_RSEQ /* diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d89d682d5337..4fe4cbf0bf08 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5683,8 +5683,8 @@ struct task_struct *idle_task(int cpu) * based on the task model parameters and gives the minimal utilization * required to meet deadlines. */ -unsigned long schedutil_cpu_util(int cpu, unsigned long util_cfs, - unsigned long max, enum schedutil_type type, +unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, + unsigned long max, enum cpu_util_type type, struct task_struct *p) { unsigned long dl_util, util, irq; @@ -5768,6 +5768,12 @@ unsigned long schedutil_cpu_util(int cpu, unsigned long util_cfs, return min(max, util); } + +unsigned long sched_cpu_util(int cpu, unsigned long max) +{ + return effective_cpu_util(cpu, cpu_util_cfs(cpu_rq(cpu)), max, + ENERGY_UTIL, NULL); +} #endif /* CONFIG_SMP */ /** diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 1dfa69246485..41e498b0008a 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -178,7 +178,7 @@ static void sugov_get_util(struct sugov_cpu *sg_cpu) sg_cpu->max = max; sg_cpu->bw_dl = cpu_bw_dl(rq); - sg_cpu->util = schedutil_cpu_util(sg_cpu->cpu, cpu_util_cfs(rq), max, + sg_cpu->util = effective_cpu_util(sg_cpu->cpu, cpu_util_cfs(rq), max, FREQUENCY_UTIL, NULL); } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 04a3ce20da67..39c5bda90bd4 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6543,7 +6543,7 @@ compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) * is already enough to scale the EM reported power * consumption at the (eventually clamped) cpu_capacity. */ - sum_util += schedutil_cpu_util(cpu, util_cfs, cpu_cap, + sum_util += effective_cpu_util(cpu, util_cfs, cpu_cap, ENERGY_UTIL, NULL); /* @@ -6553,7 +6553,7 @@ compute_energy(struct task_struct *p, int dst_cpu, struct perf_domain *pd) * NOTE: in case RT tasks are running, by default the * FREQUENCY_UTIL's utilization can be max OPP. */ - cpu_util = schedutil_cpu_util(cpu, util_cfs, cpu_cap, + cpu_util = effective_cpu_util(cpu, util_cfs, cpu_cap, FREQUENCY_UTIL, tsk); max_util = max(max_util, cpu_util); } @@ -6651,7 +6651,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) * IOW, placing the task there would make the CPU * overutilized. Take uclamp into account to see how * much capacity we can get out of the CPU; this is - * aligned with schedutil_cpu_util(). + * aligned with sched_cpu_util(). */ util = uclamp_rq_util_with(cpu_rq(cpu), util, p); if (!fits_capacity(util, cpu_cap)) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 242d4c5a5efc..045b01064c1e 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2559,22 +2559,22 @@ static inline unsigned long capacity_orig_of(int cpu) } /** - * enum schedutil_type - CPU utilization type + * enum cpu_util_type - CPU utilization type * @FREQUENCY_UTIL: Utilization used to select frequency * @ENERGY_UTIL: Utilization used during energy calculation * * The utilization signals of all scheduling classes (CFS/RT/DL) and IRQ time * need to be aggregated differently depending on the usage made of them. This - * enum is used within schedutil_freq_util() to differentiate the types of + * enum is used within effective_cpu_util() to differentiate the types of * utilization expected by the callers, and adjust the aggregation accordingly. */ -enum schedutil_type { +enum cpu_util_type { FREQUENCY_UTIL, ENERGY_UTIL, }; -unsigned long schedutil_cpu_util(int cpu, unsigned long util_cfs, - unsigned long max, enum schedutil_type type, +unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, + unsigned long max, enum cpu_util_type type, struct task_struct *p); static inline unsigned long cpu_bw_dl(struct rq *rq) -- cgit v1.2.3 From dfd5e3f5fe27bda91d5cc028c86ffbb7a0614489 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 9 Dec 2020 16:06:21 +0100 Subject: locking/lockdep: Mark local_lock_t The local_lock_t's are special, because they cannot form IRQ inversions, make sure we can tell them apart from the rest of the locks. Signed-off-by: Peter Zijlstra (Intel) --- include/linux/local_lock_internal.h | 5 ++++- include/linux/lockdep.h | 15 ++++++++++++--- include/linux/lockdep_types.h | 18 ++++++++++++++---- kernel/locking/lockdep.c | 16 +++++++++------- 4 files changed, 39 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/local_lock_internal.h b/include/linux/local_lock_internal.h index 4a8795b21d77..ded90b097e6e 100644 --- a/include/linux/local_lock_internal.h +++ b/include/linux/local_lock_internal.h @@ -18,6 +18,7 @@ typedef struct { .dep_map = { \ .name = #lockname, \ .wait_type_inner = LD_WAIT_CONFIG, \ + .lock_type = LD_LOCK_PERCPU, \ } #else # define LL_DEP_MAP_INIT(lockname) @@ -30,7 +31,9 @@ do { \ static struct lock_class_key __key; \ \ debug_check_no_locks_freed((void *)lock, sizeof(*lock));\ - lockdep_init_map_wait(&(lock)->dep_map, #lock, &__key, 0, LD_WAIT_CONFIG);\ + lockdep_init_map_type(&(lock)->dep_map, #lock, &__key, 0, \ + LD_WAIT_CONFIG, LD_WAIT_INV, \ + LD_LOCK_PERCPU); \ } while (0) #ifdef CONFIG_DEBUG_LOCK_ALLOC diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index b9e9adec73e8..7b7ebf2e28ec 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -185,12 +185,19 @@ extern void lockdep_unregister_key(struct lock_class_key *key); * to lockdep: */ -extern void lockdep_init_map_waits(struct lockdep_map *lock, const char *name, - struct lock_class_key *key, int subclass, short inner, short outer); +extern void lockdep_init_map_type(struct lockdep_map *lock, const char *name, + struct lock_class_key *key, int subclass, u8 inner, u8 outer, u8 lock_type); + +static inline void +lockdep_init_map_waits(struct lockdep_map *lock, const char *name, + struct lock_class_key *key, int subclass, u8 inner, u8 outer) +{ + lockdep_init_map_type(lock, name, key, subclass, inner, LD_WAIT_INV, LD_LOCK_NORMAL); +} static inline void lockdep_init_map_wait(struct lockdep_map *lock, const char *name, - struct lock_class_key *key, int subclass, short inner) + struct lock_class_key *key, int subclass, u8 inner) { lockdep_init_map_waits(lock, name, key, subclass, inner, LD_WAIT_INV); } @@ -340,6 +347,8 @@ static inline void lockdep_set_selftest_task(struct task_struct *task) # define lock_set_class(l, n, k, s, i) do { } while (0) # define lock_set_subclass(l, s, i) do { } while (0) # define lockdep_init() do { } while (0) +# define lockdep_init_map_type(lock, name, key, sub, inner, outer, type) \ + do { (void)(name); (void)(key); } while (0) # define lockdep_init_map_waits(lock, name, key, sub, inner, outer) \ do { (void)(name); (void)(key); } while (0) # define lockdep_init_map_wait(lock, name, key, sub, inner) \ diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h index 9a1fd49df17f..2ec9ff5a7fff 100644 --- a/include/linux/lockdep_types.h +++ b/include/linux/lockdep_types.h @@ -30,6 +30,12 @@ enum lockdep_wait_type { LD_WAIT_MAX, /* must be last */ }; +enum lockdep_lock_type { + LD_LOCK_NORMAL = 0, /* normal, catch all */ + LD_LOCK_PERCPU, /* percpu */ + LD_LOCK_MAX, +}; + #ifdef CONFIG_LOCKDEP /* @@ -119,8 +125,10 @@ struct lock_class { int name_version; const char *name; - short wait_type_inner; - short wait_type_outer; + u8 wait_type_inner; + u8 wait_type_outer; + u8 lock_type; + /* u8 hole; */ #ifdef CONFIG_LOCK_STAT unsigned long contention_point[LOCKSTAT_POINTS]; @@ -169,8 +177,10 @@ struct lockdep_map { struct lock_class_key *key; struct lock_class *class_cache[NR_LOCKDEP_CACHING_CLASSES]; const char *name; - short wait_type_outer; /* can be taken in this context */ - short wait_type_inner; /* presents this context */ + u8 wait_type_outer; /* can be taken in this context */ + u8 wait_type_inner; /* presents this context */ + u8 lock_type; + /* u8 hole; */ #ifdef CONFIG_LOCK_STAT int cpu; unsigned long ip; diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index c1418b47f625..b061e2991700 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -1290,6 +1290,7 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force) class->name_version = count_matching_names(class); class->wait_type_inner = lock->wait_type_inner; class->wait_type_outer = lock->wait_type_outer; + class->lock_type = lock->lock_type; /* * We use RCU's safe list-add method to make * parallel walking of the hash-list safe: @@ -4503,9 +4504,9 @@ print_lock_invalid_wait_context(struct task_struct *curr, */ static int check_wait_context(struct task_struct *curr, struct held_lock *next) { - short next_inner = hlock_class(next)->wait_type_inner; - short next_outer = hlock_class(next)->wait_type_outer; - short curr_inner; + u8 next_inner = hlock_class(next)->wait_type_inner; + u8 next_outer = hlock_class(next)->wait_type_outer; + u8 curr_inner; int depth; if (!curr->lockdep_depth || !next_inner || next->trylock) @@ -4528,7 +4529,7 @@ static int check_wait_context(struct task_struct *curr, struct held_lock *next) for (; depth < curr->lockdep_depth; depth++) { struct held_lock *prev = curr->held_locks + depth; - short prev_inner = hlock_class(prev)->wait_type_inner; + u8 prev_inner = hlock_class(prev)->wait_type_inner; if (prev_inner) { /* @@ -4577,9 +4578,9 @@ static inline int check_wait_context(struct task_struct *curr, /* * Initialize a lock instance's lock-class mapping info: */ -void lockdep_init_map_waits(struct lockdep_map *lock, const char *name, +void lockdep_init_map_type(struct lockdep_map *lock, const char *name, struct lock_class_key *key, int subclass, - short inner, short outer) + u8 inner, u8 outer, u8 lock_type) { int i; @@ -4602,6 +4603,7 @@ void lockdep_init_map_waits(struct lockdep_map *lock, const char *name, lock->wait_type_outer = outer; lock->wait_type_inner = inner; + lock->lock_type = lock_type; /* * No key, no joy, we need to hash something. @@ -4636,7 +4638,7 @@ void lockdep_init_map_waits(struct lockdep_map *lock, const char *name, raw_local_irq_restore(flags); } } -EXPORT_SYMBOL_GPL(lockdep_init_map_waits); +EXPORT_SYMBOL_GPL(lockdep_init_map_type); struct lock_class_key __lockdep_no_validate__; EXPORT_SYMBOL_GPL(__lockdep_no_validate__); -- cgit v1.2.3 From b1ae3587d16a8c8fc9453e147c8708d6f006ffbb Mon Sep 17 00:00:00 2001 From: Bjarni Jonasson Date: Wed, 13 Jan 2021 12:56:25 +0100 Subject: net: phy: Add 100 base-x mode Sparx-5 supports this mode and it is missing in the PHY core. Signed-off-by: Bjarni Jonasson Reviewed-by: Russell King Signed-off-by: Jakub Kicinski --- Documentation/networking/phy.rst | 5 +++++ include/linux/phy.h | 4 ++++ 2 files changed, 9 insertions(+) (limited to 'include/linux') diff --git a/Documentation/networking/phy.rst b/Documentation/networking/phy.rst index b2f7ec794bc8..399f17976a6c 100644 --- a/Documentation/networking/phy.rst +++ b/Documentation/networking/phy.rst @@ -286,6 +286,11 @@ Some of the interface modes are described below: Note: due to legacy usage, some 10GBASE-R usage incorrectly makes use of this definition. +``PHY_INTERFACE_MODE_100BASEX`` + This defines IEEE 802.3 Clause 24. The link operates at a fixed data + rate of 125Mpbs using a 4B/5B encoding scheme, resulting in an underlying + data rate of 100Mpbs. + Pause frames / flow control =========================== diff --git a/include/linux/phy.h b/include/linux/phy.h index 9effb511acde..24fcc6456a9e 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -104,6 +104,7 @@ extern const int phy_10gbit_features_array[1]; * @PHY_INTERFACE_MODE_MOCA: Multimedia over Coax * @PHY_INTERFACE_MODE_QSGMII: Quad SGMII * @PHY_INTERFACE_MODE_TRGMII: Turbo RGMII + * @PHY_INTERFACE_MODE_100BASEX: 100 BaseX * @PHY_INTERFACE_MODE_1000BASEX: 1000 BaseX * @PHY_INTERFACE_MODE_2500BASEX: 2500 BaseX * @PHY_INTERFACE_MODE_RXAUI: Reduced XAUI @@ -135,6 +136,7 @@ typedef enum { PHY_INTERFACE_MODE_MOCA, PHY_INTERFACE_MODE_QSGMII, PHY_INTERFACE_MODE_TRGMII, + PHY_INTERFACE_MODE_100BASEX, PHY_INTERFACE_MODE_1000BASEX, PHY_INTERFACE_MODE_2500BASEX, PHY_INTERFACE_MODE_RXAUI, @@ -217,6 +219,8 @@ static inline const char *phy_modes(phy_interface_t interface) return "usxgmii"; case PHY_INTERFACE_MODE_10GKR: return "10gbase-kr"; + case PHY_INTERFACE_MODE_100BASEX: + return "100base-x"; default: return "unknown"; } -- cgit v1.2.3 From 91c960b0056672e74627776655c926388350fa30 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Thu, 14 Jan 2021 18:17:44 +0000 Subject: bpf: Rename BPF_XADD and prepare to encode other atomics in .imm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A subsequent patch will add additional atomic operations. These new operations will use the same opcode field as the existing XADD, with the immediate discriminating different operations. In preparation, rename the instruction mode BPF_ATOMIC and start calling the zero immediate BPF_ADD. This is possible (doesn't break existing valid BPF progs) because the immediate field is currently reserved MBZ and BPF_ADD is zero. All uses are removed from the tree but the BPF_XADD definition is kept around to avoid breaking builds for people including kernel headers. Signed-off-by: Brendan Jackman Signed-off-by: Alexei Starovoitov Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20210114181751.768687-5-jackmanb@google.com --- Documentation/networking/filter.rst | 30 ++++++++------ arch/arm/net/bpf_jit_32.c | 7 ++-- arch/arm64/net/bpf_jit_comp.c | 16 ++++++-- arch/mips/net/ebpf_jit.c | 11 ++++-- arch/powerpc/net/bpf_jit_comp64.c | 25 +++++++++--- arch/riscv/net/bpf_jit_comp32.c | 20 ++++++++-- arch/riscv/net/bpf_jit_comp64.c | 16 ++++++-- arch/s390/net/bpf_jit_comp.c | 27 +++++++------ arch/sparc/net/bpf_jit_comp_64.c | 17 ++++++-- arch/x86/net/bpf_jit_comp.c | 46 ++++++++++++++++------ arch/x86/net/bpf_jit_comp32.c | 6 +-- drivers/net/ethernet/netronome/nfp/bpf/jit.c | 14 +++++-- drivers/net/ethernet/netronome/nfp/bpf/main.h | 4 +- drivers/net/ethernet/netronome/nfp/bpf/verifier.c | 15 ++++--- include/linux/filter.h | 16 ++++++-- include/uapi/linux/bpf.h | 5 ++- kernel/bpf/core.c | 31 ++++++++++----- kernel/bpf/disasm.c | 6 ++- kernel/bpf/verifier.c | 24 ++++++----- lib/test_bpf.c | 14 +++---- samples/bpf/bpf_insn.h | 4 +- samples/bpf/cookie_uid_helper_example.c | 8 ++-- samples/bpf/sock_example.c | 2 +- samples/bpf/test_cgrp2_attach.c | 5 ++- tools/include/linux/filter.h | 15 +++++-- tools/include/uapi/linux/bpf.h | 5 ++- .../selftests/bpf/prog_tests/cgroup_attach_multi.c | 4 +- tools/testing/selftests/bpf/test_cgroup_storage.c | 2 +- tools/testing/selftests/bpf/verifier/ctx.c | 7 ++-- .../selftests/bpf/verifier/direct_packet_access.c | 4 +- tools/testing/selftests/bpf/verifier/leak_ptr.c | 10 ++--- tools/testing/selftests/bpf/verifier/meta_access.c | 4 +- tools/testing/selftests/bpf/verifier/unpriv.c | 3 +- .../selftests/bpf/verifier/value_illegal_alu.c | 2 +- tools/testing/selftests/bpf/verifier/xadd.c | 18 ++++----- 35 files changed, 291 insertions(+), 152 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/filter.rst b/Documentation/networking/filter.rst index debb59e374de..1583d59d806d 100644 --- a/Documentation/networking/filter.rst +++ b/Documentation/networking/filter.rst @@ -1006,13 +1006,13 @@ Size modifier is one of ... Mode modifier is one of:: - BPF_IMM 0x00 /* used for 32-bit mov in classic BPF and 64-bit in eBPF */ - BPF_ABS 0x20 - BPF_IND 0x40 - BPF_MEM 0x60 - BPF_LEN 0x80 /* classic BPF only, reserved in eBPF */ - BPF_MSH 0xa0 /* classic BPF only, reserved in eBPF */ - BPF_XADD 0xc0 /* eBPF only, exclusive add */ + BPF_IMM 0x00 /* used for 32-bit mov in classic BPF and 64-bit in eBPF */ + BPF_ABS 0x20 + BPF_IND 0x40 + BPF_MEM 0x60 + BPF_LEN 0x80 /* classic BPF only, reserved in eBPF */ + BPF_MSH 0xa0 /* classic BPF only, reserved in eBPF */ + BPF_ATOMIC 0xc0 /* eBPF only, atomic operations */ eBPF has two non-generic instructions: (BPF_ABS | | BPF_LD) and (BPF_IND | | BPF_LD) which are used to access packet data. @@ -1044,11 +1044,19 @@ Unlike classic BPF instruction set, eBPF has generic load/store operations:: BPF_MEM | | BPF_STX: *(size *) (dst_reg + off) = src_reg BPF_MEM | | BPF_ST: *(size *) (dst_reg + off) = imm32 BPF_MEM | | BPF_LDX: dst_reg = *(size *) (src_reg + off) - BPF_XADD | BPF_W | BPF_STX: lock xadd *(u32 *)(dst_reg + off16) += src_reg - BPF_XADD | BPF_DW | BPF_STX: lock xadd *(u64 *)(dst_reg + off16) += src_reg -Where size is one of: BPF_B or BPF_H or BPF_W or BPF_DW. Note that 1 and -2 byte atomic increments are not supported. +Where size is one of: BPF_B or BPF_H or BPF_W or BPF_DW. + +It also includes atomic operations, which use the immediate field for extra +encoding. + + .imm = BPF_ADD, .code = BPF_ATOMIC | BPF_W | BPF_STX: lock xadd *(u32 *)(dst_reg + off16) += src_reg + .imm = BPF_ADD, .code = BPF_ATOMIC | BPF_DW | BPF_STX: lock xadd *(u64 *)(dst_reg + off16) += src_reg + +Note that 1 and 2 byte atomic operations are not supported. + +You may encounter BPF_XADD - this is a legacy name for BPF_ATOMIC, referring to +the exclusive-add operation encoded when the immediate field is zero. eBPF has one 16-byte instruction: BPF_LD | BPF_DW | BPF_IMM which consists of two consecutive ``struct bpf_insn`` 8-byte blocks and interpreted as single diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index 0207b6ea6e8a..897634d0a67c 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -1620,10 +1620,9 @@ exit: } emit_str_r(dst_lo, tmp2, off, ctx, BPF_SIZE(code)); break; - /* STX XADD: lock *(u32 *)(dst + off) += src */ - case BPF_STX | BPF_XADD | BPF_W: - /* STX XADD: lock *(u64 *)(dst + off) += src */ - case BPF_STX | BPF_XADD | BPF_DW: + /* Atomic ops */ + case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_ATOMIC | BPF_DW: goto notyet; /* STX: *(size *)(dst + off) = src */ case BPF_STX | BPF_MEM | BPF_W: diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index ef9f1d5e989d..f7b194878a99 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -875,10 +875,18 @@ emit_cond_jmp: } break; - /* STX XADD: lock *(u32 *)(dst + off) += src */ - case BPF_STX | BPF_XADD | BPF_W: - /* STX XADD: lock *(u64 *)(dst + off) += src */ - case BPF_STX | BPF_XADD | BPF_DW: + case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_ATOMIC | BPF_DW: + if (insn->imm != BPF_ADD) { + pr_err_once("unknown atomic op code %02x\n", insn->imm); + return -EINVAL; + } + + /* STX XADD: lock *(u32 *)(dst + off) += src + * and + * STX XADD: lock *(u64 *)(dst + off) += src + */ + if (!off) { reg = dst; } else { diff --git a/arch/mips/net/ebpf_jit.c b/arch/mips/net/ebpf_jit.c index 561154cbcc40..939dd06764bc 100644 --- a/arch/mips/net/ebpf_jit.c +++ b/arch/mips/net/ebpf_jit.c @@ -1423,8 +1423,8 @@ jeq_common: case BPF_STX | BPF_H | BPF_MEM: case BPF_STX | BPF_W | BPF_MEM: case BPF_STX | BPF_DW | BPF_MEM: - case BPF_STX | BPF_W | BPF_XADD: - case BPF_STX | BPF_DW | BPF_XADD: + case BPF_STX | BPF_W | BPF_ATOMIC: + case BPF_STX | BPF_DW | BPF_ATOMIC: if (insn->dst_reg == BPF_REG_10) { ctx->flags |= EBPF_SEEN_FP; dst = MIPS_R_SP; @@ -1438,7 +1438,12 @@ jeq_common: src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp); if (src < 0) return src; - if (BPF_MODE(insn->code) == BPF_XADD) { + if (BPF_MODE(insn->code) == BPF_ATOMIC) { + if (insn->imm != BPF_ADD) { + pr_err("ATOMIC OP %02x NOT HANDLED\n", insn->imm); + return -EINVAL; + } + /* * If mem_off does not fit within the 9 bit ll/sc * instruction immediate field, use a temp reg. diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 022103c6a201..aaf1a887f653 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -683,10 +683,18 @@ emit_clear: break; /* - * BPF_STX XADD (atomic_add) + * BPF_STX ATOMIC (atomic ops) */ - /* *(u32 *)(dst + off) += src */ - case BPF_STX | BPF_XADD | BPF_W: + case BPF_STX | BPF_ATOMIC | BPF_W: + if (insn->imm != BPF_ADD) { + pr_err_ratelimited( + "eBPF filter atomic op code %02x (@%d) unsupported\n", + code, i); + return -ENOTSUPP; + } + + /* *(u32 *)(dst + off) += src */ + /* Get EA into TMP_REG_1 */ EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off)); tmp_idx = ctx->idx * 4; @@ -699,8 +707,15 @@ emit_clear: /* we're done if this succeeded */ PPC_BCC_SHORT(COND_NE, tmp_idx); break; - /* *(u64 *)(dst + off) += src */ - case BPF_STX | BPF_XADD | BPF_DW: + case BPF_STX | BPF_ATOMIC | BPF_DW: + if (insn->imm != BPF_ADD) { + pr_err_ratelimited( + "eBPF filter atomic op code %02x (@%d) unsupported\n", + code, i); + return -ENOTSUPP; + } + /* *(u64 *)(dst + off) += src */ + EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off)); tmp_idx = ctx->idx * 4; EMIT(PPC_RAW_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0)); diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index 579575f9cdae..81de865f4c7c 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -881,7 +881,7 @@ static int emit_store_r64(const s8 *dst, const s8 *src, s16 off, const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); const s8 *rs = bpf_get_reg64(src, tmp2, ctx); - if (mode == BPF_XADD && size != BPF_W) + if (mode == BPF_ATOMIC && size != BPF_W) return -1; emit_imm(RV_REG_T0, off, ctx); @@ -899,7 +899,7 @@ static int emit_store_r64(const s8 *dst, const s8 *src, s16 off, case BPF_MEM: emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx); break; - case BPF_XADD: + case BPF_ATOMIC: /* Only BPF_ADD supported */ emit(rv_amoadd_w(RV_REG_ZERO, lo(rs), RV_REG_T0, 0, 0), ctx); break; @@ -1260,7 +1260,6 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_STX | BPF_MEM | BPF_H: case BPF_STX | BPF_MEM | BPF_W: case BPF_STX | BPF_MEM | BPF_DW: - case BPF_STX | BPF_XADD | BPF_W: if (BPF_CLASS(code) == BPF_ST) { emit_imm32(tmp2, imm, ctx); src = tmp2; @@ -1271,8 +1270,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, return -1; break; + case BPF_STX | BPF_ATOMIC | BPF_W: + if (insn->imm != BPF_ADD) { + pr_info_once( + "bpf-jit: not supported: atomic operation %02x ***\n", + insn->imm); + return -EFAULT; + } + + if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code), + BPF_MODE(code))) + return -1; + break; + /* No hardware support for 8-byte atomics in RV32. */ - case BPF_STX | BPF_XADD | BPF_DW: + case BPF_STX | BPF_ATOMIC | BPF_DW: /* Fallthrough. */ notsupported: diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 8a56b5293117..b44ff52f84a6 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -1027,10 +1027,18 @@ out_be: emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); emit_sd(RV_REG_T1, 0, rs, ctx); break; - /* STX XADD: lock *(u32 *)(dst + off) += src */ - case BPF_STX | BPF_XADD | BPF_W: - /* STX XADD: lock *(u64 *)(dst + off) += src */ - case BPF_STX | BPF_XADD | BPF_DW: + case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_ATOMIC | BPF_DW: + if (insn->imm != BPF_ADD) { + pr_err("bpf-jit: not supported: atomic operation %02x ***\n", + insn->imm); + return -EINVAL; + } + + /* atomic_add: lock *(u32 *)(dst + off) += src + * atomic_add: lock *(u64 *)(dst + off) += src + */ + if (off) { if (is_12b_int(off)) { emit_addi(RV_REG_T1, rd, off, ctx); diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 0a4182792876..f973e2ead197 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -1205,18 +1205,23 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, jit->seen |= SEEN_MEM; break; /* - * BPF_STX XADD (atomic_add) + * BPF_ATOMIC */ - case BPF_STX | BPF_XADD | BPF_W: /* *(u32 *)(dst + off) += src */ - /* laal %w0,%src,off(%dst) */ - EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W0, src_reg, - dst_reg, off); - jit->seen |= SEEN_MEM; - break; - case BPF_STX | BPF_XADD | BPF_DW: /* *(u64 *)(dst + off) += src */ - /* laalg %w0,%src,off(%dst) */ - EMIT6_DISP_LH(0xeb000000, 0x00ea, REG_W0, src_reg, - dst_reg, off); + case BPF_STX | BPF_ATOMIC | BPF_DW: + case BPF_STX | BPF_ATOMIC | BPF_W: + if (insn->imm != BPF_ADD) { + pr_err("Unknown atomic operation %02x\n", insn->imm); + return -1; + } + + /* *(u32/u64 *)(dst + off) += src + * + * BFW_W: laal %w0,%src,off(%dst) + * BPF_DW: laalg %w0,%src,off(%dst) + */ + EMIT6_DISP_LH(0xeb000000, + BPF_SIZE(insn->code) == BPF_W ? 0x00fa : 0x00ea, + REG_W0, src_reg, dst_reg, off); jit->seen |= SEEN_MEM; break; /* diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c index 3364e2a00989..4b8d3c65d266 100644 --- a/arch/sparc/net/bpf_jit_comp_64.c +++ b/arch/sparc/net/bpf_jit_comp_64.c @@ -1366,12 +1366,18 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) break; } - /* STX XADD: lock *(u32 *)(dst + off) += src */ - case BPF_STX | BPF_XADD | BPF_W: { + case BPF_STX | BPF_ATOMIC | BPF_W: { const u8 tmp = bpf2sparc[TMP_REG_1]; const u8 tmp2 = bpf2sparc[TMP_REG_2]; const u8 tmp3 = bpf2sparc[TMP_REG_3]; + if (insn->imm != BPF_ADD) { + pr_err_once("unknown atomic op %02x\n", insn->imm); + return -EINVAL; + } + + /* lock *(u32 *)(dst + off) += src */ + if (insn->dst_reg == BPF_REG_FP) ctx->saw_frame_pointer = true; @@ -1390,11 +1396,16 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) break; } /* STX XADD: lock *(u64 *)(dst + off) += src */ - case BPF_STX | BPF_XADD | BPF_DW: { + case BPF_STX | BPF_ATOMIC | BPF_DW: { const u8 tmp = bpf2sparc[TMP_REG_1]; const u8 tmp2 = bpf2sparc[TMP_REG_2]; const u8 tmp3 = bpf2sparc[TMP_REG_3]; + if (insn->imm != BPF_ADD) { + pr_err_once("unknown atomic op %02x\n", insn->imm); + return -EINVAL; + } + if (insn->dst_reg == BPF_REG_FP) ctx->saw_frame_pointer = true; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 93f32e0ba0ef..b1829a534da1 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -795,6 +795,33 @@ static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) *pprog = prog; } +static int emit_atomic(u8 **pprog, u8 atomic_op, + u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size) +{ + u8 *prog = *pprog; + int cnt = 0; + + EMIT1(0xF0); /* lock prefix */ + + maybe_emit_mod(&prog, dst_reg, src_reg, bpf_size == BPF_DW); + + /* emit opcode */ + switch (atomic_op) { + case BPF_ADD: + /* lock *(u32/u64*)(dst_reg + off) = src_reg */ + EMIT1(simple_alu_opcodes[atomic_op]); + break; + default: + pr_err("bpf_jit: unknown atomic opcode %02x\n", atomic_op); + return -EFAULT; + } + + emit_insn_suffix(&prog, dst_reg, src_reg, off); + + *pprog = prog; + return 0; +} + static bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs, int trapnr, unsigned long error_code, unsigned long fault_addr) @@ -839,6 +866,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, int i, cnt = 0, excnt = 0; int proglen = 0; u8 *prog = temp; + int err; detect_reg_usage(insn, insn_cnt, callee_regs_used, &tail_call_seen); @@ -1250,18 +1278,12 @@ st: if (is_imm8(insn->off)) } break; - /* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */ - case BPF_STX | BPF_XADD | BPF_W: - /* Emit 'lock add dword ptr [rax + off], eax' */ - if (is_ereg(dst_reg) || is_ereg(src_reg)) - EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01); - else - EMIT2(0xF0, 0x01); - goto xadd; - case BPF_STX | BPF_XADD | BPF_DW: - EMIT3(0xF0, add_2mod(0x48, dst_reg, src_reg), 0x01); -xadd: - emit_modrm_dstoff(&prog, dst_reg, src_reg, insn->off); + case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_ATOMIC | BPF_DW: + err = emit_atomic(&prog, insn->imm, dst_reg, src_reg, + insn->off, BPF_SIZE(insn->code)); + if (err) + return err; break; /* call */ diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c index 96fde03aa987..d17b67c69f89 100644 --- a/arch/x86/net/bpf_jit_comp32.c +++ b/arch/x86/net/bpf_jit_comp32.c @@ -2243,10 +2243,8 @@ emit_jmp: return -EFAULT; } break; - /* STX XADD: lock *(u32 *)(dst + off) += src */ - case BPF_STX | BPF_XADD | BPF_W: - /* STX XADD: lock *(u64 *)(dst + off) += src */ - case BPF_STX | BPF_XADD | BPF_DW: + case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_ATOMIC | BPF_DW: goto notyet; case BPF_JMP | BPF_EXIT: if (seen_exit) { diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c index 0a721f6e8676..e31f8fbbc696 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c @@ -3109,13 +3109,19 @@ mem_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, bool is64) return 0; } -static int mem_xadd4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) +static int mem_atomic4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) { + if (meta->insn.imm != BPF_ADD) + return -EOPNOTSUPP; + return mem_xadd(nfp_prog, meta, false); } -static int mem_xadd8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) +static int mem_atomic8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) { + if (meta->insn.imm != BPF_ADD) + return -EOPNOTSUPP; + return mem_xadd(nfp_prog, meta, true); } @@ -3475,8 +3481,8 @@ static const instr_cb_t instr_cb[256] = { [BPF_STX | BPF_MEM | BPF_H] = mem_stx2, [BPF_STX | BPF_MEM | BPF_W] = mem_stx4, [BPF_STX | BPF_MEM | BPF_DW] = mem_stx8, - [BPF_STX | BPF_XADD | BPF_W] = mem_xadd4, - [BPF_STX | BPF_XADD | BPF_DW] = mem_xadd8, + [BPF_STX | BPF_ATOMIC | BPF_W] = mem_atomic4, + [BPF_STX | BPF_ATOMIC | BPF_DW] = mem_atomic8, [BPF_ST | BPF_MEM | BPF_B] = mem_st1, [BPF_ST | BPF_MEM | BPF_H] = mem_st2, [BPF_ST | BPF_MEM | BPF_W] = mem_st4, diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.h b/drivers/net/ethernet/netronome/nfp/bpf/main.h index fac9c6f9e197..d0e17eebddd9 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/main.h +++ b/drivers/net/ethernet/netronome/nfp/bpf/main.h @@ -428,9 +428,9 @@ static inline bool is_mbpf_classic_store_pkt(const struct nfp_insn_meta *meta) return is_mbpf_classic_store(meta) && meta->ptr.type == PTR_TO_PACKET; } -static inline bool is_mbpf_xadd(const struct nfp_insn_meta *meta) +static inline bool is_mbpf_atomic(const struct nfp_insn_meta *meta) { - return (meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_XADD); + return (meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_ATOMIC); } static inline bool is_mbpf_mul(const struct nfp_insn_meta *meta) diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c index e92ee510fd52..9d235c0ce46a 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c @@ -479,7 +479,7 @@ nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, pr_vlog(env, "map writes not supported\n"); return -EOPNOTSUPP; } - if (is_mbpf_xadd(meta)) { + if (is_mbpf_atomic(meta)) { err = nfp_bpf_map_mark_used(env, meta, reg, NFP_MAP_USE_ATOMIC_CNT); if (err) @@ -523,12 +523,17 @@ exit_check_ptr: } static int -nfp_bpf_check_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, - struct bpf_verifier_env *env) +nfp_bpf_check_atomic(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, + struct bpf_verifier_env *env) { const struct bpf_reg_state *sreg = cur_regs(env) + meta->insn.src_reg; const struct bpf_reg_state *dreg = cur_regs(env) + meta->insn.dst_reg; + if (meta->insn.imm != BPF_ADD) { + pr_vlog(env, "atomic op not implemented: %d\n", meta->insn.imm); + return -EOPNOTSUPP; + } + if (dreg->type != PTR_TO_MAP_VALUE) { pr_vlog(env, "atomic add not to a map value pointer: %d\n", dreg->type); @@ -655,8 +660,8 @@ int nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, if (is_mbpf_store(meta)) return nfp_bpf_check_store(nfp_prog, meta, env); - if (is_mbpf_xadd(meta)) - return nfp_bpf_check_xadd(nfp_prog, meta, env); + if (is_mbpf_atomic(meta)) + return nfp_bpf_check_atomic(nfp_prog, meta, env); if (is_mbpf_alu(meta)) return nfp_bpf_check_alu(nfp_prog, meta, env); diff --git a/include/linux/filter.h b/include/linux/filter.h index 5edf2b660881..392e94b79668 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -259,15 +259,23 @@ static inline bool insn_is_zext(const struct bpf_insn *insn) .off = OFF, \ .imm = 0 }) -/* Atomic memory add, *(uint *)(dst_reg + off16) += src_reg */ -#define BPF_STX_XADD(SIZE, DST, SRC, OFF) \ +/* + * Atomic operations: + * + * BPF_ADD *(uint *) (dst_reg + off16) += src_reg + */ + +#define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \ ((struct bpf_insn) { \ - .code = BPF_STX | BPF_SIZE(SIZE) | BPF_XADD, \ + .code = BPF_STX | BPF_SIZE(SIZE) | BPF_ATOMIC, \ .dst_reg = DST, \ .src_reg = SRC, \ .off = OFF, \ - .imm = 0 }) + .imm = OP }) + +/* Legacy alias */ +#define BPF_STX_XADD(SIZE, DST, SRC, OFF) BPF_ATOMIC_OP(SIZE, BPF_ADD, DST, SRC, OFF) /* Memory store, *(uint *) (dst_reg + off16) = imm32 */ diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index a1ad32456f89..6b3996343e63 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -19,7 +19,8 @@ /* ld/ldx fields */ #define BPF_DW 0x18 /* double word (64-bit) */ -#define BPF_XADD 0xc0 /* exclusive add */ +#define BPF_ATOMIC 0xc0 /* atomic memory ops - op type in immediate */ +#define BPF_XADD 0xc0 /* exclusive add - legacy name */ /* alu/jmp fields */ #define BPF_MOV 0xb0 /* mov reg to reg */ @@ -2448,7 +2449,7 @@ union bpf_attr { * running simultaneously. * * A user should care about the synchronization by himself. - * For example, by using the **BPF_STX_XADD** instruction to alter + * For example, by using the **BPF_ATOMIC** instructions to alter * the shared data. * Return * A pointer to the local storage area. diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 69c3c308de5e..4836ebf459cf 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1309,8 +1309,8 @@ EXPORT_SYMBOL_GPL(__bpf_call_base); INSN_3(STX, MEM, H), \ INSN_3(STX, MEM, W), \ INSN_3(STX, MEM, DW), \ - INSN_3(STX, XADD, W), \ - INSN_3(STX, XADD, DW), \ + INSN_3(STX, ATOMIC, W), \ + INSN_3(STX, ATOMIC, DW), \ /* Immediate based. */ \ INSN_3(ST, MEM, B), \ INSN_3(ST, MEM, H), \ @@ -1618,13 +1618,25 @@ out: LDX_PROBE(DW, 8) #undef LDX_PROBE - STX_XADD_W: /* lock xadd *(u32 *)(dst_reg + off16) += src_reg */ - atomic_add((u32) SRC, (atomic_t *)(unsigned long) - (DST + insn->off)); + STX_ATOMIC_W: + switch (IMM) { + case BPF_ADD: + /* lock xadd *(u32 *)(dst_reg + off16) += src_reg */ + atomic_add((u32) SRC, (atomic_t *)(unsigned long) + (DST + insn->off)); + default: + goto default_label; + } CONT; - STX_XADD_DW: /* lock xadd *(u64 *)(dst_reg + off16) += src_reg */ - atomic64_add((u64) SRC, (atomic64_t *)(unsigned long) - (DST + insn->off)); + STX_ATOMIC_DW: + switch (IMM) { + case BPF_ADD: + /* lock xadd *(u64 *)(dst_reg + off16) += src_reg */ + atomic64_add((u64) SRC, (atomic64_t *)(unsigned long) + (DST + insn->off)); + default: + goto default_label; + } CONT; default_label: @@ -1634,7 +1646,8 @@ out: * * Note, verifier whitelists all opcodes in bpf_opcode_in_insntable(). */ - pr_warn("BPF interpreter: unknown opcode %02x\n", insn->code); + pr_warn("BPF interpreter: unknown opcode %02x (imm: 0x%x)\n", + insn->code, insn->imm); BUG_ON(1); return 0; } diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index b44d8c447afd..37c8d6e9b4cc 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -153,14 +153,16 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); - else if (BPF_MODE(insn->code) == BPF_XADD) + else if (BPF_MODE(insn->code) == BPF_ATOMIC && + insn->imm == BPF_ADD) { verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) += r%d\n", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); - else + } else { verbose(cbs->private_data, "BUG_%02x\n", insn->code); + } } else if (class == BPF_ST) { if (BPF_MODE(insn->code) != BPF_MEM) { verbose(cbs->private_data, "BUG_st_%02x\n", insn->code); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ae2aee48cf82..cfc137b81ac6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3604,13 +3604,17 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn return err; } -static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) +static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) { int err; - if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) || - insn->imm != 0) { - verbose(env, "BPF_XADD uses reserved fields\n"); + if (insn->imm != BPF_ADD) { + verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); + return -EINVAL; + } + + if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { + verbose(env, "invalid atomic operand size\n"); return -EINVAL; } @@ -3633,19 +3637,19 @@ static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_ins is_pkt_reg(env, insn->dst_reg) || is_flow_key_reg(env, insn->dst_reg) || is_sk_reg(env, insn->dst_reg)) { - verbose(env, "BPF_XADD stores into R%d %s is not allowed\n", + verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", insn->dst_reg, reg_type_str[reg_state(env, insn->dst_reg)->type]); return -EACCES; } - /* check whether atomic_add can read the memory */ + /* check whether we can read the memory */ err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_READ, -1, true); if (err) return err; - /* check whether atomic_add can write into the same memory */ + /* check whether we can write into the same memory */ return check_mem_access(env, insn_idx, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, true); } @@ -9524,8 +9528,8 @@ static int do_check(struct bpf_verifier_env *env) } else if (class == BPF_STX) { enum bpf_reg_type *prev_dst_type, dst_reg_type; - if (BPF_MODE(insn->code) == BPF_XADD) { - err = check_xadd(env, env->insn_idx, insn); + if (BPF_MODE(insn->code) == BPF_ATOMIC) { + err = check_atomic(env, env->insn_idx, insn); if (err) return err; env->insn_idx++; @@ -10010,7 +10014,7 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) if (BPF_CLASS(insn->code) == BPF_STX && ((BPF_MODE(insn->code) != BPF_MEM && - BPF_MODE(insn->code) != BPF_XADD) || insn->imm != 0)) { + BPF_MODE(insn->code) != BPF_ATOMIC) || insn->imm != 0)) { verbose(env, "BPF_STX uses reserved fields\n"); return -EINVAL; } diff --git a/lib/test_bpf.c b/lib/test_bpf.c index ca7d635bccd9..49ec9e8d8aed 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -4295,13 +4295,13 @@ static struct bpf_test tests[] = { { { 0, 0xffffffff } }, .stack_depth = 40, }, - /* BPF_STX | BPF_XADD | BPF_W/DW */ + /* BPF_STX | BPF_ATOMIC | BPF_W/DW */ { "STX_XADD_W: Test: 0x12 + 0x10 = 0x22", .u.insns_int = { BPF_ALU32_IMM(BPF_MOV, R0, 0x12), BPF_ST_MEM(BPF_W, R10, -40, 0x10), - BPF_STX_XADD(BPF_W, R10, R0, -40), + BPF_ATOMIC_OP(BPF_W, BPF_ADD, R10, R0, -40), BPF_LDX_MEM(BPF_W, R0, R10, -40), BPF_EXIT_INSN(), }, @@ -4316,7 +4316,7 @@ static struct bpf_test tests[] = { BPF_ALU64_REG(BPF_MOV, R1, R10), BPF_ALU32_IMM(BPF_MOV, R0, 0x12), BPF_ST_MEM(BPF_W, R10, -40, 0x10), - BPF_STX_XADD(BPF_W, R10, R0, -40), + BPF_ATOMIC_OP(BPF_W, BPF_ADD, R10, R0, -40), BPF_ALU64_REG(BPF_MOV, R0, R10), BPF_ALU64_REG(BPF_SUB, R0, R1), BPF_EXIT_INSN(), @@ -4331,7 +4331,7 @@ static struct bpf_test tests[] = { .u.insns_int = { BPF_ALU32_IMM(BPF_MOV, R0, 0x12), BPF_ST_MEM(BPF_W, R10, -40, 0x10), - BPF_STX_XADD(BPF_W, R10, R0, -40), + BPF_ATOMIC_OP(BPF_W, BPF_ADD, R10, R0, -40), BPF_EXIT_INSN(), }, INTERNAL, @@ -4352,7 +4352,7 @@ static struct bpf_test tests[] = { .u.insns_int = { BPF_ALU32_IMM(BPF_MOV, R0, 0x12), BPF_ST_MEM(BPF_DW, R10, -40, 0x10), - BPF_STX_XADD(BPF_DW, R10, R0, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, R10, R0, -40), BPF_LDX_MEM(BPF_DW, R0, R10, -40), BPF_EXIT_INSN(), }, @@ -4367,7 +4367,7 @@ static struct bpf_test tests[] = { BPF_ALU64_REG(BPF_MOV, R1, R10), BPF_ALU32_IMM(BPF_MOV, R0, 0x12), BPF_ST_MEM(BPF_DW, R10, -40, 0x10), - BPF_STX_XADD(BPF_DW, R10, R0, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, R10, R0, -40), BPF_ALU64_REG(BPF_MOV, R0, R10), BPF_ALU64_REG(BPF_SUB, R0, R1), BPF_EXIT_INSN(), @@ -4382,7 +4382,7 @@ static struct bpf_test tests[] = { .u.insns_int = { BPF_ALU32_IMM(BPF_MOV, R0, 0x12), BPF_ST_MEM(BPF_DW, R10, -40, 0x10), - BPF_STX_XADD(BPF_DW, R10, R0, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, R10, R0, -40), BPF_EXIT_INSN(), }, INTERNAL, diff --git a/samples/bpf/bpf_insn.h b/samples/bpf/bpf_insn.h index 544237980582..db67a2847395 100644 --- a/samples/bpf/bpf_insn.h +++ b/samples/bpf/bpf_insn.h @@ -138,11 +138,11 @@ struct bpf_insn; #define BPF_STX_XADD(SIZE, DST, SRC, OFF) \ ((struct bpf_insn) { \ - .code = BPF_STX | BPF_SIZE(SIZE) | BPF_XADD, \ + .code = BPF_STX | BPF_SIZE(SIZE) | BPF_ATOMIC, \ .dst_reg = DST, \ .src_reg = SRC, \ .off = OFF, \ - .imm = 0 }) + .imm = BPF_ADD }) /* Memory store, *(uint *) (dst_reg + off16) = imm32 */ diff --git a/samples/bpf/cookie_uid_helper_example.c b/samples/bpf/cookie_uid_helper_example.c index deb0e3e0324d..c5ff7a13918c 100644 --- a/samples/bpf/cookie_uid_helper_example.c +++ b/samples/bpf/cookie_uid_helper_example.c @@ -147,12 +147,12 @@ static void prog_load(void) */ BPF_MOV64_REG(BPF_REG_9, BPF_REG_0), BPF_MOV64_IMM(BPF_REG_1, 1), - BPF_STX_XADD(BPF_DW, BPF_REG_9, BPF_REG_1, - offsetof(struct stats, packets)), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_9, BPF_REG_1, + offsetof(struct stats, packets)), BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6, offsetof(struct __sk_buff, len)), - BPF_STX_XADD(BPF_DW, BPF_REG_9, BPF_REG_1, - offsetof(struct stats, bytes)), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_9, BPF_REG_1, + offsetof(struct stats, bytes)), BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, offsetof(struct __sk_buff, len)), BPF_EXIT_INSN(), diff --git a/samples/bpf/sock_example.c b/samples/bpf/sock_example.c index 00aae1d33fca..23d1930e1927 100644 --- a/samples/bpf/sock_example.c +++ b/samples/bpf/sock_example.c @@ -54,7 +54,7 @@ static int test_sock(void) BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2), BPF_MOV64_IMM(BPF_REG_1, 1), /* r1 = 1 */ - BPF_RAW_INSN(BPF_STX | BPF_XADD | BPF_DW, BPF_REG_0, BPF_REG_1, 0, 0), /* xadd r0 += r1 */ + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_0, BPF_REG_1, 0), BPF_MOV64_IMM(BPF_REG_0, 0), /* r0 = 0 */ BPF_EXIT_INSN(), }; diff --git a/samples/bpf/test_cgrp2_attach.c b/samples/bpf/test_cgrp2_attach.c index 20fbd1241db3..390ff38d2ac6 100644 --- a/samples/bpf/test_cgrp2_attach.c +++ b/samples/bpf/test_cgrp2_attach.c @@ -53,7 +53,7 @@ static int prog_load(int map_fd, int verdict) BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2), BPF_MOV64_IMM(BPF_REG_1, 1), /* r1 = 1 */ - BPF_RAW_INSN(BPF_STX | BPF_XADD | BPF_DW, BPF_REG_0, BPF_REG_1, 0, 0), /* xadd r0 += r1 */ + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_0, BPF_REG_1, 0), /* Count bytes */ BPF_MOV64_IMM(BPF_REG_0, MAP_KEY_BYTES), /* r0 = 1 */ @@ -64,7 +64,8 @@ static int prog_load(int map_fd, int verdict) BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2), BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6, offsetof(struct __sk_buff, len)), /* r1 = skb->len */ - BPF_RAW_INSN(BPF_STX | BPF_XADD | BPF_DW, BPF_REG_0, BPF_REG_1, 0, 0), /* xadd r0 += r1 */ + + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_0, BPF_REG_1, 0), BPF_MOV64_IMM(BPF_REG_0, verdict), /* r0 = verdict */ BPF_EXIT_INSN(), diff --git a/tools/include/linux/filter.h b/tools/include/linux/filter.h index ca28b6ab8db7..e870c9039f0d 100644 --- a/tools/include/linux/filter.h +++ b/tools/include/linux/filter.h @@ -169,15 +169,22 @@ .off = OFF, \ .imm = 0 }) -/* Atomic memory add, *(uint *)(dst_reg + off16) += src_reg */ +/* + * Atomic operations: + * + * BPF_ADD *(uint *) (dst_reg + off16) += src_reg + */ -#define BPF_STX_XADD(SIZE, DST, SRC, OFF) \ +#define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \ ((struct bpf_insn) { \ - .code = BPF_STX | BPF_SIZE(SIZE) | BPF_XADD, \ + .code = BPF_STX | BPF_SIZE(SIZE) | BPF_ATOMIC, \ .dst_reg = DST, \ .src_reg = SRC, \ .off = OFF, \ - .imm = 0 }) + .imm = OP }) + +/* Legacy alias */ +#define BPF_STX_XADD(SIZE, DST, SRC, OFF) BPF_ATOMIC_OP(SIZE, BPF_ADD, DST, SRC, OFF) /* Memory store, *(uint *) (dst_reg + off16) = imm32 */ diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index a1ad32456f89..6b3996343e63 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -19,7 +19,8 @@ /* ld/ldx fields */ #define BPF_DW 0x18 /* double word (64-bit) */ -#define BPF_XADD 0xc0 /* exclusive add */ +#define BPF_ATOMIC 0xc0 /* atomic memory ops - op type in immediate */ +#define BPF_XADD 0xc0 /* exclusive add - legacy name */ /* alu/jmp fields */ #define BPF_MOV 0xb0 /* mov reg to reg */ @@ -2448,7 +2449,7 @@ union bpf_attr { * running simultaneously. * * A user should care about the synchronization by himself. - * For example, by using the **BPF_STX_XADD** instruction to alter + * For example, by using the **BPF_ATOMIC** instructions to alter * the shared data. * Return * A pointer to the local storage area. diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_attach_multi.c b/tools/testing/selftests/bpf/prog_tests/cgroup_attach_multi.c index b549fcfacc0b..0a1fc9816cef 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_attach_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_attach_multi.c @@ -45,13 +45,13 @@ static int prog_load_cnt(int verdict, int val) BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2), BPF_MOV64_IMM(BPF_REG_1, val), /* r1 = 1 */ - BPF_RAW_INSN(BPF_STX | BPF_XADD | BPF_DW, BPF_REG_0, BPF_REG_1, 0, 0), /* xadd r0 += r1 */ + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_0, BPF_REG_1, 0), BPF_LD_MAP_FD(BPF_REG_1, cgroup_storage_fd), BPF_MOV64_IMM(BPF_REG_2, 0), BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_local_storage), BPF_MOV64_IMM(BPF_REG_1, val), - BPF_RAW_INSN(BPF_STX | BPF_XADD | BPF_W, BPF_REG_0, BPF_REG_1, 0, 0), + BPF_ATOMIC_OP(BPF_W, BPF_ADD, BPF_REG_0, BPF_REG_1, 0), BPF_LD_MAP_FD(BPF_REG_1, percpu_cgroup_storage_fd), BPF_MOV64_IMM(BPF_REG_2, 0), diff --git a/tools/testing/selftests/bpf/test_cgroup_storage.c b/tools/testing/selftests/bpf/test_cgroup_storage.c index d946252a25bb..0cda61da5d39 100644 --- a/tools/testing/selftests/bpf/test_cgroup_storage.c +++ b/tools/testing/selftests/bpf/test_cgroup_storage.c @@ -29,7 +29,7 @@ int main(int argc, char **argv) BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_local_storage), BPF_MOV64_IMM(BPF_REG_1, 1), - BPF_STX_XADD(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_0, BPF_REG_1, 0), BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0), BPF_ALU64_IMM(BPF_AND, BPF_REG_1, 0x1), BPF_MOV64_REG(BPF_REG_0, BPF_REG_1), diff --git a/tools/testing/selftests/bpf/verifier/ctx.c b/tools/testing/selftests/bpf/verifier/ctx.c index 93d6b1641481..23080862aafd 100644 --- a/tools/testing/selftests/bpf/verifier/ctx.c +++ b/tools/testing/selftests/bpf/verifier/ctx.c @@ -10,14 +10,13 @@ .prog_type = BPF_PROG_TYPE_SCHED_CLS, }, { - "context stores via XADD", + "context stores via BPF_ATOMIC", .insns = { BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_RAW_INSN(BPF_STX | BPF_XADD | BPF_W, BPF_REG_1, - BPF_REG_0, offsetof(struct __sk_buff, mark), 0), + BPF_ATOMIC_OP(BPF_W, BPF_ADD, BPF_REG_1, BPF_REG_0, offsetof(struct __sk_buff, mark)), BPF_EXIT_INSN(), }, - .errstr = "BPF_XADD stores into R1 ctx is not allowed", + .errstr = "BPF_ATOMIC stores into R1 ctx is not allowed", .result = REJECT, .prog_type = BPF_PROG_TYPE_SCHED_CLS, }, diff --git a/tools/testing/selftests/bpf/verifier/direct_packet_access.c b/tools/testing/selftests/bpf/verifier/direct_packet_access.c index ae72536603fe..ac1e19d0f520 100644 --- a/tools/testing/selftests/bpf/verifier/direct_packet_access.c +++ b/tools/testing/selftests/bpf/verifier/direct_packet_access.c @@ -333,7 +333,7 @@ BPF_MOV64_REG(BPF_REG_4, BPF_REG_10), BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -8), BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0), - BPF_STX_XADD(BPF_DW, BPF_REG_4, BPF_REG_5, 0), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_4, BPF_REG_5, 0), BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_4, 0), BPF_STX_MEM(BPF_W, BPF_REG_2, BPF_REG_5, 0), BPF_MOV64_IMM(BPF_REG_0, 0), @@ -488,7 +488,7 @@ BPF_JMP_REG(BPF_JGT, BPF_REG_0, BPF_REG_3, 11), BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, -8), BPF_MOV64_IMM(BPF_REG_4, 0xffffffff), - BPF_STX_XADD(BPF_DW, BPF_REG_10, BPF_REG_4, -8), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_10, BPF_REG_4, -8), BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_10, -8), BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 49), BPF_ALU64_REG(BPF_ADD, BPF_REG_4, BPF_REG_2), diff --git a/tools/testing/selftests/bpf/verifier/leak_ptr.c b/tools/testing/selftests/bpf/verifier/leak_ptr.c index d6eec17f2cd2..73f0dea95546 100644 --- a/tools/testing/selftests/bpf/verifier/leak_ptr.c +++ b/tools/testing/selftests/bpf/verifier/leak_ptr.c @@ -5,7 +5,7 @@ BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, offsetof(struct __sk_buff, cb[0])), BPF_LD_MAP_FD(BPF_REG_2, 0), - BPF_STX_XADD(BPF_DW, BPF_REG_1, BPF_REG_2, + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_1, BPF_REG_2, offsetof(struct __sk_buff, cb[0])), BPF_EXIT_INSN(), }, @@ -13,7 +13,7 @@ .errstr_unpriv = "R2 leaks addr into mem", .result_unpriv = REJECT, .result = REJECT, - .errstr = "BPF_XADD stores into R1 ctx is not allowed", + .errstr = "BPF_ATOMIC stores into R1 ctx is not allowed", }, { "leak pointer into ctx 2", @@ -21,14 +21,14 @@ BPF_MOV64_IMM(BPF_REG_0, 0), BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, offsetof(struct __sk_buff, cb[0])), - BPF_STX_XADD(BPF_DW, BPF_REG_1, BPF_REG_10, + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_1, BPF_REG_10, offsetof(struct __sk_buff, cb[0])), BPF_EXIT_INSN(), }, .errstr_unpriv = "R10 leaks addr into mem", .result_unpriv = REJECT, .result = REJECT, - .errstr = "BPF_XADD stores into R1 ctx is not allowed", + .errstr = "BPF_ATOMIC stores into R1 ctx is not allowed", }, { "leak pointer into ctx 3", @@ -56,7 +56,7 @@ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3), BPF_MOV64_IMM(BPF_REG_3, 0), BPF_STX_MEM(BPF_DW, BPF_REG_0, BPF_REG_3, 0), - BPF_STX_XADD(BPF_DW, BPF_REG_0, BPF_REG_6, 0), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_0, BPF_REG_6, 0), BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }, diff --git a/tools/testing/selftests/bpf/verifier/meta_access.c b/tools/testing/selftests/bpf/verifier/meta_access.c index 205292b8dd65..b45e8af41420 100644 --- a/tools/testing/selftests/bpf/verifier/meta_access.c +++ b/tools/testing/selftests/bpf/verifier/meta_access.c @@ -171,7 +171,7 @@ BPF_MOV64_IMM(BPF_REG_5, 42), BPF_MOV64_IMM(BPF_REG_6, 24), BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_5, -8), - BPF_STX_XADD(BPF_DW, BPF_REG_10, BPF_REG_6, -8), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_10, BPF_REG_6, -8), BPF_LDX_MEM(BPF_DW, BPF_REG_5, BPF_REG_10, -8), BPF_JMP_IMM(BPF_JGT, BPF_REG_5, 100, 6), BPF_ALU64_REG(BPF_ADD, BPF_REG_3, BPF_REG_5), @@ -196,7 +196,7 @@ BPF_MOV64_IMM(BPF_REG_5, 42), BPF_MOV64_IMM(BPF_REG_6, 24), BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_5, -8), - BPF_STX_XADD(BPF_DW, BPF_REG_10, BPF_REG_6, -8), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_10, BPF_REG_6, -8), BPF_LDX_MEM(BPF_DW, BPF_REG_5, BPF_REG_10, -8), BPF_JMP_IMM(BPF_JGT, BPF_REG_5, 100, 6), BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_5), diff --git a/tools/testing/selftests/bpf/verifier/unpriv.c b/tools/testing/selftests/bpf/verifier/unpriv.c index a3fe0fbaed41..ee298627abae 100644 --- a/tools/testing/selftests/bpf/verifier/unpriv.c +++ b/tools/testing/selftests/bpf/verifier/unpriv.c @@ -207,7 +207,8 @@ BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, -8), BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0), BPF_MOV64_IMM(BPF_REG_0, 1), - BPF_RAW_INSN(BPF_STX | BPF_XADD | BPF_DW, BPF_REG_10, BPF_REG_0, -8, 0), + BPF_RAW_INSN(BPF_STX | BPF_ATOMIC | BPF_DW, + BPF_REG_10, BPF_REG_0, -8, BPF_ADD), BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, 0), BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_hash_recalc), BPF_EXIT_INSN(), diff --git a/tools/testing/selftests/bpf/verifier/value_illegal_alu.c b/tools/testing/selftests/bpf/verifier/value_illegal_alu.c index ed1c2cea1dea..489062867218 100644 --- a/tools/testing/selftests/bpf/verifier/value_illegal_alu.c +++ b/tools/testing/selftests/bpf/verifier/value_illegal_alu.c @@ -82,7 +82,7 @@ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8), BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_0, 0), - BPF_STX_XADD(BPF_DW, BPF_REG_2, BPF_REG_3, 0), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_2, BPF_REG_3, 0), BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0), BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 22), BPF_EXIT_INSN(), diff --git a/tools/testing/selftests/bpf/verifier/xadd.c b/tools/testing/selftests/bpf/verifier/xadd.c index c5de2e62cc8b..b96ef3526815 100644 --- a/tools/testing/selftests/bpf/verifier/xadd.c +++ b/tools/testing/selftests/bpf/verifier/xadd.c @@ -3,7 +3,7 @@ .insns = { BPF_MOV64_IMM(BPF_REG_0, 1), BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8), - BPF_STX_XADD(BPF_W, BPF_REG_10, BPF_REG_0, -7), + BPF_ATOMIC_OP(BPF_W, BPF_ADD, BPF_REG_10, BPF_REG_0, -7), BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -8), BPF_EXIT_INSN(), }, @@ -22,7 +22,7 @@ BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1), BPF_EXIT_INSN(), BPF_MOV64_IMM(BPF_REG_1, 1), - BPF_STX_XADD(BPF_W, BPF_REG_0, BPF_REG_1, 3), + BPF_ATOMIC_OP(BPF_W, BPF_ADD, BPF_REG_0, BPF_REG_1, 3), BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 3), BPF_EXIT_INSN(), }, @@ -45,13 +45,13 @@ BPF_MOV64_IMM(BPF_REG_0, 1), BPF_ST_MEM(BPF_W, BPF_REG_2, 0, 0), BPF_ST_MEM(BPF_W, BPF_REG_2, 3, 0), - BPF_STX_XADD(BPF_W, BPF_REG_2, BPF_REG_0, 1), - BPF_STX_XADD(BPF_W, BPF_REG_2, BPF_REG_0, 2), + BPF_ATOMIC_OP(BPF_W, BPF_ADD, BPF_REG_2, BPF_REG_0, 1), + BPF_ATOMIC_OP(BPF_W, BPF_ADD, BPF_REG_2, BPF_REG_0, 2), BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_2, 1), BPF_EXIT_INSN(), }, .result = REJECT, - .errstr = "BPF_XADD stores into R2 pkt is not allowed", + .errstr = "BPF_ATOMIC stores into R2 pkt is not allowed", .prog_type = BPF_PROG_TYPE_XDP, .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, @@ -62,8 +62,8 @@ BPF_MOV64_REG(BPF_REG_6, BPF_REG_0), BPF_MOV64_REG(BPF_REG_7, BPF_REG_10), BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8), - BPF_STX_XADD(BPF_DW, BPF_REG_10, BPF_REG_0, -8), - BPF_STX_XADD(BPF_DW, BPF_REG_10, BPF_REG_0, -8), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_10, BPF_REG_0, -8), + BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_10, BPF_REG_0, -8), BPF_JMP_REG(BPF_JNE, BPF_REG_6, BPF_REG_0, 3), BPF_JMP_REG(BPF_JNE, BPF_REG_7, BPF_REG_10, 2), BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -8), @@ -82,8 +82,8 @@ BPF_MOV64_REG(BPF_REG_6, BPF_REG_0), BPF_MOV64_REG(BPF_REG_7, BPF_REG_10), BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -8), - BPF_STX_XADD(BPF_W, BPF_REG_10, BPF_REG_0, -8), - BPF_STX_XADD(BPF_W, BPF_REG_10, BPF_REG_0, -8), + BPF_ATOMIC_OP(BPF_W, BPF_ADD, BPF_REG_10, BPF_REG_0, -8), + BPF_ATOMIC_OP(BPF_W, BPF_ADD, BPF_REG_10, BPF_REG_0, -8), BPF_JMP_REG(BPF_JNE, BPF_REG_6, BPF_REG_0, 3), BPF_JMP_REG(BPF_JNE, BPF_REG_7, BPF_REG_10, 2), BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_10, -8), -- cgit v1.2.3 From 5ca419f2864a2c60940dcf4bbaeb69546200e36f Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Thu, 14 Jan 2021 18:17:46 +0000 Subject: bpf: Add BPF_FETCH field / create atomic_fetch_add instruction The BPF_FETCH field can be set in bpf_insn.imm, for BPF_ATOMIC instructions, in order to have the previous value of the atomically-modified memory location loaded into the src register after an atomic op is carried out. Suggested-by: Yonghong Song Signed-off-by: Brendan Jackman Signed-off-by: Alexei Starovoitov Acked-by: John Fastabend Link: https://lore.kernel.org/bpf/20210114181751.768687-7-jackmanb@google.com --- arch/x86/net/bpf_jit_comp.c | 4 ++++ include/linux/filter.h | 1 + include/uapi/linux/bpf.h | 3 +++ kernel/bpf/core.c | 13 +++++++++++++ kernel/bpf/disasm.c | 7 +++++++ kernel/bpf/verifier.c | 33 ++++++++++++++++++++++++--------- tools/include/linux/filter.h | 1 + tools/include/uapi/linux/bpf.h | 3 +++ 8 files changed, 56 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index b1829a534da1..eea7d8b0bb12 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -811,6 +811,10 @@ static int emit_atomic(u8 **pprog, u8 atomic_op, /* lock *(u32/u64*)(dst_reg + off) = src_reg */ EMIT1(simple_alu_opcodes[atomic_op]); break; + case BPF_ADD | BPF_FETCH: + /* src_reg = atomic_fetch_add(dst_reg + off, src_reg); */ + EMIT2(0x0F, 0xC1); + break; default: pr_err("bpf_jit: unknown atomic opcode %02x\n", atomic_op); return -EFAULT; diff --git a/include/linux/filter.h b/include/linux/filter.h index 392e94b79668..23fca41b8540 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -264,6 +264,7 @@ static inline bool insn_is_zext(const struct bpf_insn *insn) * Atomic operations: * * BPF_ADD *(uint *) (dst_reg + off16) += src_reg + * BPF_ADD | BPF_FETCH src_reg = atomic_fetch_add(dst_reg + off16, src_reg); */ #define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \ diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 6b3996343e63..ea262b009049 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -44,6 +44,9 @@ #define BPF_CALL 0x80 /* function call */ #define BPF_EXIT 0x90 /* function return */ +/* atomic op type fields (stored in immediate) */ +#define BPF_FETCH 0x01 /* fetch previous value into src reg */ + /* Register numbers */ enum { BPF_REG_0 = 0, diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 4836ebf459cf..28d6000463e4 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1624,16 +1624,29 @@ out: /* lock xadd *(u32 *)(dst_reg + off16) += src_reg */ atomic_add((u32) SRC, (atomic_t *)(unsigned long) (DST + insn->off)); + break; + case BPF_ADD | BPF_FETCH: + SRC = (u32) atomic_fetch_add( + (u32) SRC, + (atomic_t *)(unsigned long) (DST + insn->off)); + break; default: goto default_label; } CONT; + STX_ATOMIC_DW: switch (IMM) { case BPF_ADD: /* lock xadd *(u64 *)(dst_reg + off16) += src_reg */ atomic64_add((u64) SRC, (atomic64_t *)(unsigned long) (DST + insn->off)); + break; + case BPF_ADD | BPF_FETCH: + SRC = (u64) atomic64_fetch_add( + (u64) SRC, + (atomic64_t *)(unsigned long) (DST + insn->off)); + break; default: goto default_label; } diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 37c8d6e9b4cc..d2e20f6d0516 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -160,6 +160,13 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); + } else if (BPF_MODE(insn->code) == BPF_ATOMIC && + insn->imm == (BPF_ADD | BPF_FETCH)) { + verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_add((%s *)(r%d %+d), r%d)\n", + insn->code, insn->src_reg, + BPF_SIZE(insn->code) == BPF_DW ? "64" : "", + bpf_ldst_string[BPF_SIZE(insn->code) >> 3], + insn->dst_reg, insn->off, insn->src_reg); } else { verbose(cbs->private_data, "BUG_%02x\n", insn->code); } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d8a85f4e5b95..6aa1fc919761 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3608,7 +3608,11 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i { int err; - if (insn->imm != BPF_ADD) { + switch (insn->imm) { + case BPF_ADD: + case BPF_ADD | BPF_FETCH: + break; + default: verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); return -EINVAL; } @@ -3650,8 +3654,20 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i return err; /* check whether we can write into the same memory */ - return check_mem_access(env, insn_idx, insn->dst_reg, insn->off, - BPF_SIZE(insn->code), BPF_WRITE, -1, true); + err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, + BPF_SIZE(insn->code), BPF_WRITE, -1, true); + if (err) + return err; + + if (!(insn->imm & BPF_FETCH)) + return 0; + + /* check and record load of old value into src reg */ + err = check_reg_arg(env, insn->src_reg, DST_OP); + if (err) + return err; + + return 0; } static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno, @@ -9528,12 +9544,6 @@ static int do_check(struct bpf_verifier_env *env) } else if (class == BPF_STX) { enum bpf_reg_type *prev_dst_type, dst_reg_type; - if (((BPF_MODE(insn->code) != BPF_MEM && - BPF_MODE(insn->code) != BPF_ATOMIC) || insn->imm != 0)) { - verbose(env, "BPF_STX uses reserved fields\n"); - return -EINVAL; - } - if (BPF_MODE(insn->code) == BPF_ATOMIC) { err = check_atomic(env, env->insn_idx, insn); if (err) @@ -9542,6 +9552,11 @@ static int do_check(struct bpf_verifier_env *env) continue; } + if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { + verbose(env, "BPF_STX uses reserved fields\n"); + return -EINVAL; + } + /* check src1 operand */ err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) diff --git a/tools/include/linux/filter.h b/tools/include/linux/filter.h index e870c9039f0d..7211ce9fba53 100644 --- a/tools/include/linux/filter.h +++ b/tools/include/linux/filter.h @@ -173,6 +173,7 @@ * Atomic operations: * * BPF_ADD *(uint *) (dst_reg + off16) += src_reg + * BPF_ADD | BPF_FETCH src_reg = atomic_fetch_add(dst_reg + off16, src_reg); */ #define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \ diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 6b3996343e63..ea262b009049 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -44,6 +44,9 @@ #define BPF_CALL 0x80 /* function call */ #define BPF_EXIT 0x90 /* function return */ +/* atomic op type fields (stored in immediate) */ +#define BPF_FETCH 0x01 /* fetch previous value into src reg */ + /* Register numbers */ enum { BPF_REG_0 = 0, -- cgit v1.2.3 From 5ffa25502b5ab3d639829a2d1e316cff7f59a41e Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Thu, 14 Jan 2021 18:17:47 +0000 Subject: bpf: Add instructions for atomic_[cmp]xchg This adds two atomic opcodes, both of which include the BPF_FETCH flag. XCHG without the BPF_FETCH flag would naturally encode atomic_set. This is not supported because it would be of limited value to userspace (it doesn't imply any barriers). CMPXCHG without BPF_FETCH woulud be an atomic compare-and-write. We don't have such an operation in the kernel so it isn't provided to BPF either. There are two significant design decisions made for the CMPXCHG instruction: - To solve the issue that this operation fundamentally has 3 operands, but we only have two register fields. Therefore the operand we compare against (the kernel's API calls it 'old') is hard-coded to be R0. x86 has similar design (and A64 doesn't have this problem). A potential alternative might be to encode the other operand's register number in the immediate field. - The kernel's atomic_cmpxchg returns the old value, while the C11 userspace APIs return a boolean indicating the comparison result. Which should BPF do? A64 returns the old value. x86 returns the old value in the hard-coded register (and also sets a flag). That means return-old-value is easier to JIT, so that's what we use. Signed-off-by: Brendan Jackman Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20210114181751.768687-8-jackmanb@google.com --- arch/x86/net/bpf_jit_comp.c | 8 ++++++++ include/linux/filter.h | 2 ++ include/uapi/linux/bpf.h | 4 +++- kernel/bpf/core.c | 20 ++++++++++++++++++++ kernel/bpf/disasm.c | 15 +++++++++++++++ kernel/bpf/verifier.c | 19 +++++++++++++++++-- tools/include/linux/filter.h | 2 ++ tools/include/uapi/linux/bpf.h | 4 +++- 8 files changed, 70 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index eea7d8b0bb12..308241187582 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -815,6 +815,14 @@ static int emit_atomic(u8 **pprog, u8 atomic_op, /* src_reg = atomic_fetch_add(dst_reg + off, src_reg); */ EMIT2(0x0F, 0xC1); break; + case BPF_XCHG: + /* src_reg = atomic_xchg(dst_reg + off, src_reg); */ + EMIT1(0x87); + break; + case BPF_CMPXCHG: + /* r0 = atomic_cmpxchg(dst_reg + off, r0, src_reg); */ + EMIT2(0x0F, 0xB1); + break; default: pr_err("bpf_jit: unknown atomic opcode %02x\n", atomic_op); return -EFAULT; diff --git a/include/linux/filter.h b/include/linux/filter.h index 23fca41b8540..d563820f197d 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -265,6 +265,8 @@ static inline bool insn_is_zext(const struct bpf_insn *insn) * * BPF_ADD *(uint *) (dst_reg + off16) += src_reg * BPF_ADD | BPF_FETCH src_reg = atomic_fetch_add(dst_reg + off16, src_reg); + * BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg) + * BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg) */ #define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \ diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index ea262b009049..c001766adcbc 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -45,7 +45,9 @@ #define BPF_EXIT 0x90 /* function return */ /* atomic op type fields (stored in immediate) */ -#define BPF_FETCH 0x01 /* fetch previous value into src reg */ +#define BPF_FETCH 0x01 /* not an opcode on its own, used to build others */ +#define BPF_XCHG (0xe0 | BPF_FETCH) /* atomic exchange */ +#define BPF_CMPXCHG (0xf0 | BPF_FETCH) /* atomic compare-and-write */ /* Register numbers */ enum { diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 28d6000463e4..4df6daba43ef 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1630,6 +1630,16 @@ out: (u32) SRC, (atomic_t *)(unsigned long) (DST + insn->off)); break; + case BPF_XCHG: + SRC = (u32) atomic_xchg( + (atomic_t *)(unsigned long) (DST + insn->off), + (u32) SRC); + break; + case BPF_CMPXCHG: + BPF_R0 = (u32) atomic_cmpxchg( + (atomic_t *)(unsigned long) (DST + insn->off), + (u32) BPF_R0, (u32) SRC); + break; default: goto default_label; } @@ -1647,6 +1657,16 @@ out: (u64) SRC, (atomic64_t *)(unsigned long) (DST + insn->off)); break; + case BPF_XCHG: + SRC = (u64) atomic64_xchg( + (atomic64_t *)(unsigned long) (DST + insn->off), + (u64) SRC); + break; + case BPF_CMPXCHG: + BPF_R0 = (u64) atomic64_cmpxchg( + (atomic64_t *)(unsigned long) (DST + insn->off), + (u64) BPF_R0, (u64) SRC); + break; default: goto default_label; } diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index d2e20f6d0516..ee8d1132767b 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -167,6 +167,21 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); + } else if (BPF_MODE(insn->code) == BPF_ATOMIC && + insn->imm == BPF_CMPXCHG) { + verbose(cbs->private_data, "(%02x) r0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)\n", + insn->code, + BPF_SIZE(insn->code) == BPF_DW ? "64" : "", + bpf_ldst_string[BPF_SIZE(insn->code) >> 3], + insn->dst_reg, insn->off, + insn->src_reg); + } else if (BPF_MODE(insn->code) == BPF_ATOMIC && + insn->imm == BPF_XCHG) { + verbose(cbs->private_data, "(%02x) r%d = atomic%s_xchg((%s *)(r%d %+d), r%d)\n", + insn->code, insn->src_reg, + BPF_SIZE(insn->code) == BPF_DW ? "64" : "", + bpf_ldst_string[BPF_SIZE(insn->code) >> 3], + insn->dst_reg, insn->off, insn->src_reg); } else { verbose(cbs->private_data, "BUG_%02x\n", insn->code); } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6aa1fc919761..89a4d154ab37 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3606,11 +3606,14 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) { + int load_reg; int err; switch (insn->imm) { case BPF_ADD: case BPF_ADD | BPF_FETCH: + case BPF_XCHG: + case BPF_CMPXCHG: break; default: verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); @@ -3632,6 +3635,13 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i if (err) return err; + if (insn->imm == BPF_CMPXCHG) { + /* Check comparison of R0 with memory location */ + err = check_reg_arg(env, BPF_REG_0, SRC_OP); + if (err) + return err; + } + if (is_pointer_value(env, insn->src_reg)) { verbose(env, "R%d leaks addr into mem\n", insn->src_reg); return -EACCES; @@ -3662,8 +3672,13 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i if (!(insn->imm & BPF_FETCH)) return 0; - /* check and record load of old value into src reg */ - err = check_reg_arg(env, insn->src_reg, DST_OP); + if (insn->imm == BPF_CMPXCHG) + load_reg = BPF_REG_0; + else + load_reg = insn->src_reg; + + /* check and record load of old value */ + err = check_reg_arg(env, load_reg, DST_OP); if (err) return err; diff --git a/tools/include/linux/filter.h b/tools/include/linux/filter.h index 7211ce9fba53..d75998b0d5ac 100644 --- a/tools/include/linux/filter.h +++ b/tools/include/linux/filter.h @@ -174,6 +174,8 @@ * * BPF_ADD *(uint *) (dst_reg + off16) += src_reg * BPF_ADD | BPF_FETCH src_reg = atomic_fetch_add(dst_reg + off16, src_reg); + * BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg) + * BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg) */ #define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \ diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index ea262b009049..c001766adcbc 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -45,7 +45,9 @@ #define BPF_EXIT 0x90 /* function return */ /* atomic op type fields (stored in immediate) */ -#define BPF_FETCH 0x01 /* fetch previous value into src reg */ +#define BPF_FETCH 0x01 /* not an opcode on its own, used to build others */ +#define BPF_XCHG (0xe0 | BPF_FETCH) /* atomic exchange */ +#define BPF_CMPXCHG (0xf0 | BPF_FETCH) /* atomic compare-and-write */ /* Register numbers */ enum { -- cgit v1.2.3 From 981f94c3e92146705baf97fb417a5ed1ab1a79a5 Mon Sep 17 00:00:00 2001 From: Brendan Jackman Date: Thu, 14 Jan 2021 18:17:49 +0000 Subject: bpf: Add bitwise atomic instructions This adds instructions for atomic[64]_[fetch_]and atomic[64]_[fetch_]or atomic[64]_[fetch_]xor All these operations are isomorphic enough to implement with the same verifier, interpreter, and x86 JIT code, hence being a single commit. The main interesting thing here is that x86 doesn't directly support the fetch_ version these operations, so we need to generate a CMPXCHG loop in the JIT. This requires the use of two temporary registers, IIUC it's safe to use BPF_REG_AX and x86's AUX_REG for this purpose. Signed-off-by: Brendan Jackman Signed-off-by: Alexei Starovoitov Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20210114181751.768687-10-jackmanb@google.com --- arch/x86/net/bpf_jit_comp.c | 50 +++++++++++++++++++++++++++++++++++++++++++- include/linux/filter.h | 6 ++++++ kernel/bpf/core.c | 3 +++ kernel/bpf/disasm.c | 21 +++++++++++++++---- kernel/bpf/verifier.c | 6 ++++++ tools/include/linux/filter.h | 6 ++++++ 6 files changed, 87 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 308241187582..1d4d50199293 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -808,6 +808,10 @@ static int emit_atomic(u8 **pprog, u8 atomic_op, /* emit opcode */ switch (atomic_op) { case BPF_ADD: + case BPF_SUB: + case BPF_AND: + case BPF_OR: + case BPF_XOR: /* lock *(u32/u64*)(dst_reg + off) = src_reg */ EMIT1(simple_alu_opcodes[atomic_op]); break; @@ -1292,8 +1296,52 @@ st: if (is_imm8(insn->off)) case BPF_STX | BPF_ATOMIC | BPF_W: case BPF_STX | BPF_ATOMIC | BPF_DW: + if (insn->imm == (BPF_AND | BPF_FETCH) || + insn->imm == (BPF_OR | BPF_FETCH) || + insn->imm == (BPF_XOR | BPF_FETCH)) { + u8 *branch_target; + bool is64 = BPF_SIZE(insn->code) == BPF_DW; + + /* + * Can't be implemented with a single x86 insn. + * Need to do a CMPXCHG loop. + */ + + /* Will need RAX as a CMPXCHG operand so save R0 */ + emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); + branch_target = prog; + /* Load old value */ + emit_ldx(&prog, BPF_SIZE(insn->code), + BPF_REG_0, dst_reg, insn->off); + /* + * Perform the (commutative) operation locally, + * put the result in the AUX_REG. + */ + emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); + maybe_emit_mod(&prog, AUX_REG, src_reg, is64); + EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)], + add_2reg(0xC0, AUX_REG, src_reg)); + /* Attempt to swap in new value */ + err = emit_atomic(&prog, BPF_CMPXCHG, + dst_reg, AUX_REG, insn->off, + BPF_SIZE(insn->code)); + if (WARN_ON(err)) + return err; + /* + * ZF tells us whether we won the race. If it's + * cleared we need to try again. + */ + EMIT2(X86_JNE, -(prog - branch_target) - 2); + /* Return the pre-modification value */ + emit_mov_reg(&prog, is64, src_reg, BPF_REG_0); + /* Restore R0 after clobbering RAX */ + emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); + break; + + } + err = emit_atomic(&prog, insn->imm, dst_reg, src_reg, - insn->off, BPF_SIZE(insn->code)); + insn->off, BPF_SIZE(insn->code)); if (err) return err; break; diff --git a/include/linux/filter.h b/include/linux/filter.h index d563820f197d..7fdce5407214 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -264,7 +264,13 @@ static inline bool insn_is_zext(const struct bpf_insn *insn) * Atomic operations: * * BPF_ADD *(uint *) (dst_reg + off16) += src_reg + * BPF_AND *(uint *) (dst_reg + off16) &= src_reg + * BPF_OR *(uint *) (dst_reg + off16) |= src_reg + * BPF_XOR *(uint *) (dst_reg + off16) ^= src_reg * BPF_ADD | BPF_FETCH src_reg = atomic_fetch_add(dst_reg + off16, src_reg); + * BPF_AND | BPF_FETCH src_reg = atomic_fetch_and(dst_reg + off16, src_reg); + * BPF_OR | BPF_FETCH src_reg = atomic_fetch_or(dst_reg + off16, src_reg); + * BPF_XOR | BPF_FETCH src_reg = atomic_fetch_xor(dst_reg + off16, src_reg); * BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg) * BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg) */ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 8669e685825f..5bbd4884ff7a 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1642,6 +1642,9 @@ out: STX_ATOMIC_W: switch (IMM) { ATOMIC_ALU_OP(BPF_ADD, add) + ATOMIC_ALU_OP(BPF_AND, and) + ATOMIC_ALU_OP(BPF_OR, or) + ATOMIC_ALU_OP(BPF_XOR, xor) #undef ATOMIC_ALU_OP case BPF_XCHG: diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index ee8d1132767b..19ff8fed7f4b 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -80,6 +80,13 @@ const char *const bpf_alu_string[16] = { [BPF_END >> 4] = "endian", }; +static const char *const bpf_atomic_alu_string[16] = { + [BPF_ADD >> 4] = "add", + [BPF_AND >> 4] = "and", + [BPF_OR >> 4] = "or", + [BPF_XOR >> 4] = "or", +}; + static const char *const bpf_ldst_string[] = { [BPF_W >> 3] = "u32", [BPF_H >> 3] = "u16", @@ -154,17 +161,23 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->dst_reg, insn->off, insn->src_reg); else if (BPF_MODE(insn->code) == BPF_ATOMIC && - insn->imm == BPF_ADD) { - verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) += r%d\n", + (insn->imm == BPF_ADD || insn->imm == BPF_ADD || + insn->imm == BPF_OR || insn->imm == BPF_XOR)) { + verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d\n", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, + bpf_alu_string[BPF_OP(insn->imm) >> 4], insn->src_reg); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && - insn->imm == (BPF_ADD | BPF_FETCH)) { - verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_add((%s *)(r%d %+d), r%d)\n", + (insn->imm == (BPF_ADD | BPF_FETCH) || + insn->imm == (BPF_AND | BPF_FETCH) || + insn->imm == (BPF_OR | BPF_FETCH) || + insn->imm == (BPF_XOR | BPF_FETCH))) { + verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)\n", insn->code, insn->src_reg, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", + bpf_atomic_alu_string[BPF_OP(insn->imm) >> 4], bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 89a4d154ab37..0f82d5d46e2c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3612,6 +3612,12 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i switch (insn->imm) { case BPF_ADD: case BPF_ADD | BPF_FETCH: + case BPF_AND: + case BPF_AND | BPF_FETCH: + case BPF_OR: + case BPF_OR | BPF_FETCH: + case BPF_XOR: + case BPF_XOR | BPF_FETCH: case BPF_XCHG: case BPF_CMPXCHG: break; diff --git a/tools/include/linux/filter.h b/tools/include/linux/filter.h index d75998b0d5ac..736bdeccdfe4 100644 --- a/tools/include/linux/filter.h +++ b/tools/include/linux/filter.h @@ -173,7 +173,13 @@ * Atomic operations: * * BPF_ADD *(uint *) (dst_reg + off16) += src_reg + * BPF_AND *(uint *) (dst_reg + off16) &= src_reg + * BPF_OR *(uint *) (dst_reg + off16) |= src_reg + * BPF_XOR *(uint *) (dst_reg + off16) ^= src_reg * BPF_ADD | BPF_FETCH src_reg = atomic_fetch_add(dst_reg + off16, src_reg); + * BPF_AND | BPF_FETCH src_reg = atomic_fetch_and(dst_reg + off16, src_reg); + * BPF_OR | BPF_FETCH src_reg = atomic_fetch_or(dst_reg + off16, src_reg); + * BPF_XOR | BPF_FETCH src_reg = atomic_fetch_xor(dst_reg + off16, src_reg); * BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg) * BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg) */ -- cgit v1.2.3 From bd7525dacd7e204f8cae061941fb9001c89d6988 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Thu, 14 Jan 2021 14:40:42 +0100 Subject: bpf: Move stack_map_get_build_id into lib Moving stack_map_get_build_id into lib with declaration in linux/buildid.h header: int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id); This function returns build id for given struct vm_area_struct. There is no functional change to stack_map_get_build_id function. Signed-off-by: Jiri Olsa Signed-off-by: Alexei Starovoitov Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20210114134044.1418404-2-jolsa@kernel.org --- include/linux/buildid.h | 11 ++++ kernel/bpf/stackmap.c | 143 ++---------------------------------------------- lib/Makefile | 3 +- lib/buildid.c | 136 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 140 deletions(-) create mode 100644 include/linux/buildid.h create mode 100644 lib/buildid.c (limited to 'include/linux') diff --git a/include/linux/buildid.h b/include/linux/buildid.h new file mode 100644 index 000000000000..08028a212589 --- /dev/null +++ b/include/linux/buildid.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_BUILDID_H +#define _LINUX_BUILDID_H + +#include + +#define BUILD_ID_SIZE_MAX 20 + +int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id); + +#endif diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index aea96b638473..55d254a59f07 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -7,10 +7,9 @@ #include #include #include -#include -#include #include #include +#include #include "percpu_freelist.h" #define STACK_CREATE_FLAG_MASK \ @@ -143,140 +142,6 @@ free_smap: return ERR_PTR(err); } -#define BPF_BUILD_ID 3 -/* - * Parse build id from the note segment. This logic can be shared between - * 32-bit and 64-bit system, because Elf32_Nhdr and Elf64_Nhdr are - * identical. - */ -static inline int stack_map_parse_build_id(void *page_addr, - unsigned char *build_id, - void *note_start, - Elf32_Word note_size) -{ - Elf32_Word note_offs = 0, new_offs; - - /* check for overflow */ - if (note_start < page_addr || note_start + note_size < note_start) - return -EINVAL; - - /* only supports note that fits in the first page */ - if (note_start + note_size > page_addr + PAGE_SIZE) - return -EINVAL; - - while (note_offs + sizeof(Elf32_Nhdr) < note_size) { - Elf32_Nhdr *nhdr = (Elf32_Nhdr *)(note_start + note_offs); - - if (nhdr->n_type == BPF_BUILD_ID && - nhdr->n_namesz == sizeof("GNU") && - nhdr->n_descsz > 0 && - nhdr->n_descsz <= BPF_BUILD_ID_SIZE) { - memcpy(build_id, - note_start + note_offs + - ALIGN(sizeof("GNU"), 4) + sizeof(Elf32_Nhdr), - nhdr->n_descsz); - memset(build_id + nhdr->n_descsz, 0, - BPF_BUILD_ID_SIZE - nhdr->n_descsz); - return 0; - } - new_offs = note_offs + sizeof(Elf32_Nhdr) + - ALIGN(nhdr->n_namesz, 4) + ALIGN(nhdr->n_descsz, 4); - if (new_offs <= note_offs) /* overflow */ - break; - note_offs = new_offs; - } - return -EINVAL; -} - -/* Parse build ID from 32-bit ELF */ -static int stack_map_get_build_id_32(void *page_addr, - unsigned char *build_id) -{ - Elf32_Ehdr *ehdr = (Elf32_Ehdr *)page_addr; - Elf32_Phdr *phdr; - int i; - - /* only supports phdr that fits in one page */ - if (ehdr->e_phnum > - (PAGE_SIZE - sizeof(Elf32_Ehdr)) / sizeof(Elf32_Phdr)) - return -EINVAL; - - phdr = (Elf32_Phdr *)(page_addr + sizeof(Elf32_Ehdr)); - - for (i = 0; i < ehdr->e_phnum; ++i) { - if (phdr[i].p_type == PT_NOTE && - !stack_map_parse_build_id(page_addr, build_id, - page_addr + phdr[i].p_offset, - phdr[i].p_filesz)) - return 0; - } - return -EINVAL; -} - -/* Parse build ID from 64-bit ELF */ -static int stack_map_get_build_id_64(void *page_addr, - unsigned char *build_id) -{ - Elf64_Ehdr *ehdr = (Elf64_Ehdr *)page_addr; - Elf64_Phdr *phdr; - int i; - - /* only supports phdr that fits in one page */ - if (ehdr->e_phnum > - (PAGE_SIZE - sizeof(Elf64_Ehdr)) / sizeof(Elf64_Phdr)) - return -EINVAL; - - phdr = (Elf64_Phdr *)(page_addr + sizeof(Elf64_Ehdr)); - - for (i = 0; i < ehdr->e_phnum; ++i) { - if (phdr[i].p_type == PT_NOTE && - !stack_map_parse_build_id(page_addr, build_id, - page_addr + phdr[i].p_offset, - phdr[i].p_filesz)) - return 0; - } - return -EINVAL; -} - -/* Parse build ID of ELF file mapped to vma */ -static int stack_map_get_build_id(struct vm_area_struct *vma, - unsigned char *build_id) -{ - Elf32_Ehdr *ehdr; - struct page *page; - void *page_addr; - int ret; - - /* only works for page backed storage */ - if (!vma->vm_file) - return -EINVAL; - - page = find_get_page(vma->vm_file->f_mapping, 0); - if (!page) - return -EFAULT; /* page not mapped */ - - ret = -EINVAL; - page_addr = kmap_atomic(page); - ehdr = (Elf32_Ehdr *)page_addr; - - /* compare magic x7f "ELF" */ - if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0) - goto out; - - /* only support executable file and shared object file */ - if (ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN) - goto out; - - if (ehdr->e_ident[EI_CLASS] == ELFCLASS32) - ret = stack_map_get_build_id_32(page_addr, build_id); - else if (ehdr->e_ident[EI_CLASS] == ELFCLASS64) - ret = stack_map_get_build_id_64(page_addr, build_id); -out: - kunmap_atomic(page_addr); - put_page(page); - return ret; -} - static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, u64 *ips, u32 trace_nr, bool user) { @@ -317,18 +182,18 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, for (i = 0; i < trace_nr; i++) { id_offs[i].status = BPF_STACK_BUILD_ID_IP; id_offs[i].ip = ips[i]; - memset(id_offs[i].build_id, 0, BPF_BUILD_ID_SIZE); + memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX); } return; } for (i = 0; i < trace_nr; i++) { vma = find_vma(current->mm, ips[i]); - if (!vma || stack_map_get_build_id(vma, id_offs[i].build_id)) { + if (!vma || build_id_parse(vma, id_offs[i].build_id)) { /* per entry fall back to ips */ id_offs[i].status = BPF_STACK_BUILD_ID_IP; id_offs[i].ip = ips[i]; - memset(id_offs[i].build_id, 0, BPF_BUILD_ID_SIZE); + memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX); continue; } id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ips[i] diff --git a/lib/Makefile b/lib/Makefile index afeff05fa8c5..a6b160c3a4fa 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -36,7 +36,8 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \ flex_proportions.o ratelimit.o show_mem.o \ is_single_threaded.o plist.o decompress.o kobject_uevent.o \ earlycpio.o seq_buf.o siphash.o dec_and_lock.o \ - nmi_backtrace.o nodemask.o win_minmax.o memcat_p.o + nmi_backtrace.o nodemask.o win_minmax.o memcat_p.o \ + buildid.o lib-$(CONFIG_PRINTK) += dump_stack.o lib-$(CONFIG_SMP) += cpumask.o diff --git a/lib/buildid.c b/lib/buildid.c new file mode 100644 index 000000000000..4a4f520c0e29 --- /dev/null +++ b/lib/buildid.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#define BUILD_ID 3 +/* + * Parse build id from the note segment. This logic can be shared between + * 32-bit and 64-bit system, because Elf32_Nhdr and Elf64_Nhdr are + * identical. + */ +static inline int parse_build_id(void *page_addr, + unsigned char *build_id, + void *note_start, + Elf32_Word note_size) +{ + Elf32_Word note_offs = 0, new_offs; + + /* check for overflow */ + if (note_start < page_addr || note_start + note_size < note_start) + return -EINVAL; + + /* only supports note that fits in the first page */ + if (note_start + note_size > page_addr + PAGE_SIZE) + return -EINVAL; + + while (note_offs + sizeof(Elf32_Nhdr) < note_size) { + Elf32_Nhdr *nhdr = (Elf32_Nhdr *)(note_start + note_offs); + + if (nhdr->n_type == BUILD_ID && + nhdr->n_namesz == sizeof("GNU") && + nhdr->n_descsz > 0 && + nhdr->n_descsz <= BUILD_ID_SIZE_MAX) { + memcpy(build_id, + note_start + note_offs + + ALIGN(sizeof("GNU"), 4) + sizeof(Elf32_Nhdr), + nhdr->n_descsz); + memset(build_id + nhdr->n_descsz, 0, + BUILD_ID_SIZE_MAX - nhdr->n_descsz); + return 0; + } + new_offs = note_offs + sizeof(Elf32_Nhdr) + + ALIGN(nhdr->n_namesz, 4) + ALIGN(nhdr->n_descsz, 4); + if (new_offs <= note_offs) /* overflow */ + break; + note_offs = new_offs; + } + return -EINVAL; +} + +/* Parse build ID from 32-bit ELF */ +static int get_build_id_32(void *page_addr, unsigned char *build_id) +{ + Elf32_Ehdr *ehdr = (Elf32_Ehdr *)page_addr; + Elf32_Phdr *phdr; + int i; + + /* only supports phdr that fits in one page */ + if (ehdr->e_phnum > + (PAGE_SIZE - sizeof(Elf32_Ehdr)) / sizeof(Elf32_Phdr)) + return -EINVAL; + + phdr = (Elf32_Phdr *)(page_addr + sizeof(Elf32_Ehdr)); + + for (i = 0; i < ehdr->e_phnum; ++i) { + if (phdr[i].p_type == PT_NOTE && + !parse_build_id(page_addr, build_id, + page_addr + phdr[i].p_offset, + phdr[i].p_filesz)) + return 0; + } + return -EINVAL; +} + +/* Parse build ID from 64-bit ELF */ +static int get_build_id_64(void *page_addr, unsigned char *build_id) +{ + Elf64_Ehdr *ehdr = (Elf64_Ehdr *)page_addr; + Elf64_Phdr *phdr; + int i; + + /* only supports phdr that fits in one page */ + if (ehdr->e_phnum > + (PAGE_SIZE - sizeof(Elf64_Ehdr)) / sizeof(Elf64_Phdr)) + return -EINVAL; + + phdr = (Elf64_Phdr *)(page_addr + sizeof(Elf64_Ehdr)); + + for (i = 0; i < ehdr->e_phnum; ++i) { + if (phdr[i].p_type == PT_NOTE && + !parse_build_id(page_addr, build_id, + page_addr + phdr[i].p_offset, + phdr[i].p_filesz)) + return 0; + } + return -EINVAL; +} + +/* Parse build ID of ELF file mapped to vma */ +int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id) +{ + Elf32_Ehdr *ehdr; + struct page *page; + void *page_addr; + int ret; + + /* only works for page backed storage */ + if (!vma->vm_file) + return -EINVAL; + + page = find_get_page(vma->vm_file->f_mapping, 0); + if (!page) + return -EFAULT; /* page not mapped */ + + ret = -EINVAL; + page_addr = kmap_atomic(page); + ehdr = (Elf32_Ehdr *)page_addr; + + /* compare magic x7f "ELF" */ + if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0) + goto out; + + /* only support executable file and shared object file */ + if (ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN) + goto out; + + if (ehdr->e_ident[EI_CLASS] == ELFCLASS32) + ret = get_build_id_32(page_addr, build_id); + else if (ehdr->e_ident[EI_CLASS] == ELFCLASS64) + ret = get_build_id_64(page_addr, build_id); +out: + kunmap_atomic(page_addr); + put_page(page); + return ret; +} -- cgit v1.2.3 From 921f88fc891922b325b3668cd026a386571ed602 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Thu, 14 Jan 2021 14:40:43 +0100 Subject: bpf: Add size arg to build_id_parse function It's possible to have other build id types (other than default SHA1). Currently there's also ld support for MD5 build id. Adding size argument to build_id_parse function, that returns (if defined) size of the parsed build id, so we can recognize the build id type. Signed-off-by: Jiri Olsa Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20210114134044.1418404-3-jolsa@kernel.org --- include/linux/buildid.h | 3 ++- kernel/bpf/stackmap.c | 2 +- lib/buildid.c | 29 +++++++++++++++++++++-------- 3 files changed, 24 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/buildid.h b/include/linux/buildid.h index 08028a212589..40232f90db6e 100644 --- a/include/linux/buildid.h +++ b/include/linux/buildid.h @@ -6,6 +6,7 @@ #define BUILD_ID_SIZE_MAX 20 -int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id); +int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, + __u32 *size); #endif diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 55d254a59f07..cabaf7db8efc 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -189,7 +189,7 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, for (i = 0; i < trace_nr; i++) { vma = find_vma(current->mm, ips[i]); - if (!vma || build_id_parse(vma, id_offs[i].build_id)) { + if (!vma || build_id_parse(vma, id_offs[i].build_id, NULL)) { /* per entry fall back to ips */ id_offs[i].status = BPF_STACK_BUILD_ID_IP; id_offs[i].ip = ips[i]; diff --git a/lib/buildid.c b/lib/buildid.c index 4a4f520c0e29..6156997c3895 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -12,6 +12,7 @@ */ static inline int parse_build_id(void *page_addr, unsigned char *build_id, + __u32 *size, void *note_start, Elf32_Word note_size) { @@ -38,6 +39,8 @@ static inline int parse_build_id(void *page_addr, nhdr->n_descsz); memset(build_id + nhdr->n_descsz, 0, BUILD_ID_SIZE_MAX - nhdr->n_descsz); + if (size) + *size = nhdr->n_descsz; return 0; } new_offs = note_offs + sizeof(Elf32_Nhdr) + @@ -50,7 +53,8 @@ static inline int parse_build_id(void *page_addr, } /* Parse build ID from 32-bit ELF */ -static int get_build_id_32(void *page_addr, unsigned char *build_id) +static int get_build_id_32(void *page_addr, unsigned char *build_id, + __u32 *size) { Elf32_Ehdr *ehdr = (Elf32_Ehdr *)page_addr; Elf32_Phdr *phdr; @@ -65,7 +69,7 @@ static int get_build_id_32(void *page_addr, unsigned char *build_id) for (i = 0; i < ehdr->e_phnum; ++i) { if (phdr[i].p_type == PT_NOTE && - !parse_build_id(page_addr, build_id, + !parse_build_id(page_addr, build_id, size, page_addr + phdr[i].p_offset, phdr[i].p_filesz)) return 0; @@ -74,7 +78,8 @@ static int get_build_id_32(void *page_addr, unsigned char *build_id) } /* Parse build ID from 64-bit ELF */ -static int get_build_id_64(void *page_addr, unsigned char *build_id) +static int get_build_id_64(void *page_addr, unsigned char *build_id, + __u32 *size) { Elf64_Ehdr *ehdr = (Elf64_Ehdr *)page_addr; Elf64_Phdr *phdr; @@ -89,7 +94,7 @@ static int get_build_id_64(void *page_addr, unsigned char *build_id) for (i = 0; i < ehdr->e_phnum; ++i) { if (phdr[i].p_type == PT_NOTE && - !parse_build_id(page_addr, build_id, + !parse_build_id(page_addr, build_id, size, page_addr + phdr[i].p_offset, phdr[i].p_filesz)) return 0; @@ -97,8 +102,16 @@ static int get_build_id_64(void *page_addr, unsigned char *build_id) return -EINVAL; } -/* Parse build ID of ELF file mapped to vma */ -int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id) +/* + * Parse build ID of ELF file mapped to vma + * @vma: vma object + * @build_id: buffer to store build id, at least BUILD_ID_SIZE long + * @size: returns actual build id size in case of success + * + * Returns 0 on success, otherwise error (< 0). + */ +int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, + __u32 *size) { Elf32_Ehdr *ehdr; struct page *page; @@ -126,9 +139,9 @@ int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id) goto out; if (ehdr->e_ident[EI_CLASS] == ELFCLASS32) - ret = get_build_id_32(page_addr, build_id); + ret = get_build_id_32(page_addr, build_id, size); else if (ehdr->e_ident[EI_CLASS] == ELFCLASS64) - ret = get_build_id_64(page_addr, build_id); + ret = get_build_id_64(page_addr, build_id, size); out: kunmap_atomic(page_addr); put_page(page); -- cgit v1.2.3 From 3c15e00e7b58bc2b37e53d2612f0a0163281be77 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Thu, 26 Nov 2020 11:41:42 +0100 Subject: mfd/bus: sunxi-rsb: Make .remove() callback return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver core ignores the return value of struct device_driver::remove because there is only little that can be done. To simplify the quest to make this function return void, let struct sunxi_rsb_driver::remove return void, too. All users already unconditionally return 0, this commit makes this obvious and ensures future users don't behave differently. To simplify even further, make axp20x_device_remove() return void instead of returning 0 unconditionally, too. Signed-off-by: Uwe Kleine-König Reviewed-by: Chen-Yu Tsai Signed-off-by: Lee Jones --- drivers/bus/sunxi-rsb.c | 4 +++- drivers/mfd/axp20x-i2c.c | 4 +++- drivers/mfd/axp20x-rsb.c | 4 ++-- drivers/mfd/axp20x.c | 4 +--- include/linux/mfd/axp20x.h | 2 +- include/linux/sunxi-rsb.h | 2 +- 6 files changed, 11 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/bus/sunxi-rsb.c b/drivers/bus/sunxi-rsb.c index 1bb00a959c67..117716e23ffb 100644 --- a/drivers/bus/sunxi-rsb.c +++ b/drivers/bus/sunxi-rsb.c @@ -170,7 +170,9 @@ static int sunxi_rsb_device_remove(struct device *dev) { const struct sunxi_rsb_driver *drv = to_sunxi_rsb_driver(dev->driver); - return drv->remove(to_sunxi_rsb_device(dev)); + drv->remove(to_sunxi_rsb_device(dev)); + + return 0; } static struct bus_type sunxi_rsb_bus = { diff --git a/drivers/mfd/axp20x-i2c.c b/drivers/mfd/axp20x-i2c.c index 2cfde81f5fbf..00ab48018d8d 100644 --- a/drivers/mfd/axp20x-i2c.c +++ b/drivers/mfd/axp20x-i2c.c @@ -54,7 +54,9 @@ static int axp20x_i2c_remove(struct i2c_client *i2c) { struct axp20x_dev *axp20x = i2c_get_clientdata(i2c); - return axp20x_device_remove(axp20x); + axp20x_device_remove(axp20x); + + return 0; } #ifdef CONFIG_OF diff --git a/drivers/mfd/axp20x-rsb.c b/drivers/mfd/axp20x-rsb.c index 4cdc79f5cc48..214bc0d84d44 100644 --- a/drivers/mfd/axp20x-rsb.c +++ b/drivers/mfd/axp20x-rsb.c @@ -49,11 +49,11 @@ static int axp20x_rsb_probe(struct sunxi_rsb_device *rdev) return axp20x_device_probe(axp20x); } -static int axp20x_rsb_remove(struct sunxi_rsb_device *rdev) +static void axp20x_rsb_remove(struct sunxi_rsb_device *rdev) { struct axp20x_dev *axp20x = sunxi_rsb_device_get_drvdata(rdev); - return axp20x_device_remove(axp20x); + axp20x_device_remove(axp20x); } static const struct of_device_id axp20x_rsb_of_match[] = { diff --git a/drivers/mfd/axp20x.c b/drivers/mfd/axp20x.c index aa59496e4376..3eae04e24ac8 100644 --- a/drivers/mfd/axp20x.c +++ b/drivers/mfd/axp20x.c @@ -987,7 +987,7 @@ int axp20x_device_probe(struct axp20x_dev *axp20x) } EXPORT_SYMBOL(axp20x_device_probe); -int axp20x_device_remove(struct axp20x_dev *axp20x) +void axp20x_device_remove(struct axp20x_dev *axp20x) { if (axp20x == axp20x_pm_power_off) { axp20x_pm_power_off = NULL; @@ -996,8 +996,6 @@ int axp20x_device_remove(struct axp20x_dev *axp20x) mfd_remove_devices(axp20x->dev); regmap_del_irq_chip(axp20x->irq, axp20x->regmap_irqc); - - return 0; } EXPORT_SYMBOL(axp20x_device_remove); diff --git a/include/linux/mfd/axp20x.h b/include/linux/mfd/axp20x.h index fd5957c042da..9ab0e2fca7ea 100644 --- a/include/linux/mfd/axp20x.h +++ b/include/linux/mfd/axp20x.h @@ -696,6 +696,6 @@ int axp20x_device_probe(struct axp20x_dev *axp20x); * * This tells the axp20x core to remove the associated mfd devices */ -int axp20x_device_remove(struct axp20x_dev *axp20x); +void axp20x_device_remove(struct axp20x_dev *axp20x); #endif /* __LINUX_MFD_AXP20X_H */ diff --git a/include/linux/sunxi-rsb.h b/include/linux/sunxi-rsb.h index 7e75bb0346d0..bf0d365f471c 100644 --- a/include/linux/sunxi-rsb.h +++ b/include/linux/sunxi-rsb.h @@ -59,7 +59,7 @@ static inline void sunxi_rsb_device_set_drvdata(struct sunxi_rsb_device *rdev, struct sunxi_rsb_driver { struct device_driver driver; int (*probe)(struct sunxi_rsb_device *rdev); - int (*remove)(struct sunxi_rsb_device *rdev); + void (*remove)(struct sunxi_rsb_device *rdev); }; static inline struct sunxi_rsb_driver *to_sunxi_rsb_driver(struct device_driver *d) -- cgit v1.2.3 From e68d0119e3284334de5650a1ac42ef4e179f895e Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Fri, 15 Jan 2021 12:49:11 +0300 Subject: software node: Introduce device_add_software_node() This helper will register a software node and then assign it to device at the same time. The function will also make sure that the device can't have more than one software node. Acked-by: Felipe Balbi Signed-off-by: Heikki Krogerus Link: https://lore.kernel.org/r/20210115094914.88401-2-heikki.krogerus@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/swnode.c | 71 ++++++++++++++++++++++++++++++++++++++++++------ include/linux/property.h | 3 ++ 2 files changed, 65 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c index 4a4b2008fbc2..4842dd678200 100644 --- a/drivers/base/swnode.c +++ b/drivers/base/swnode.c @@ -48,6 +48,19 @@ EXPORT_SYMBOL_GPL(is_software_node); struct swnode, fwnode) : NULL; \ }) +static inline struct swnode *dev_to_swnode(struct device *dev) +{ + struct fwnode_handle *fwnode = dev_fwnode(dev); + + if (!fwnode) + return NULL; + + if (!is_software_node(fwnode)) + fwnode = fwnode->secondary; + + return to_swnode(fwnode); +} + static struct swnode * software_node_to_swnode(const struct software_node *node) { @@ -843,22 +856,62 @@ void fwnode_remove_software_node(struct fwnode_handle *fwnode) } EXPORT_SYMBOL_GPL(fwnode_remove_software_node); +/** + * device_add_software_node - Assign software node to a device + * @dev: The device the software node is meant for. + * @swnode: The software node. + * + * This function will register @swnode and make it the secondary firmware node + * pointer of @dev. If @dev has no primary node, then @swnode will become the primary + * node. + */ +int device_add_software_node(struct device *dev, const struct software_node *swnode) +{ + int ret; + + /* Only one software node per device. */ + if (dev_to_swnode(dev)) + return -EBUSY; + + ret = software_node_register(swnode); + if (ret) + return ret; + + set_secondary_fwnode(dev, software_node_fwnode(swnode)); + + return 0; +} +EXPORT_SYMBOL_GPL(device_add_software_node); + +/** + * device_remove_software_node - Remove device's software node + * @dev: The device with the software node. + * + * This function will unregister the software node of @dev. + */ +void device_remove_software_node(struct device *dev) +{ + struct swnode *swnode; + + swnode = dev_to_swnode(dev); + if (!swnode) + return; + + software_node_notify(dev, KOBJ_REMOVE); + set_secondary_fwnode(dev, NULL); + kobject_put(&swnode->kobj); +} +EXPORT_SYMBOL_GPL(device_remove_software_node); + int software_node_notify(struct device *dev, unsigned long action) { - struct fwnode_handle *fwnode = dev_fwnode(dev); struct swnode *swnode; int ret; - if (!fwnode) - return 0; - - if (!is_software_node(fwnode)) - fwnode = fwnode->secondary; - if (!is_software_node(fwnode)) + swnode = dev_to_swnode(dev); + if (!swnode) return 0; - swnode = to_swnode(fwnode); - switch (action) { case KOBJ_ADD: ret = sysfs_create_link(&dev->kobj, &swnode->kobj, diff --git a/include/linux/property.h b/include/linux/property.h index 0a9001fe7aea..b0e413dc5927 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -488,4 +488,7 @@ fwnode_create_software_node(const struct property_entry *properties, const struct fwnode_handle *parent); void fwnode_remove_software_node(struct fwnode_handle *fwnode); +int device_add_software_node(struct device *dev, const struct software_node *swnode); +void device_remove_software_node(struct device *dev); + #endif /* _LINUX_PROPERTY_H_ */ -- cgit v1.2.3 From 429b29aef7f841086949c7359f9c3ccb051e7ea3 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Fri, 15 Jan 2021 16:51:29 +0100 Subject: tty: serial: Drop unused efm32 serial driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support for this machine was just removed, so drop the now unused UART driver, too. Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20210115155130.185010-7-u.kleine-koenig@pengutronix.de Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/Kconfig | 13 - drivers/tty/serial/Makefile | 1 - drivers/tty/serial/efm32-uart.c | 852 ------------------------------- include/linux/platform_data/efm32-uart.h | 19 - include/uapi/linux/serial_core.h | 3 - 5 files changed, 888 deletions(-) delete mode 100644 drivers/tty/serial/efm32-uart.c delete mode 100644 include/linux/platform_data/efm32-uart.h (limited to 'include/linux') diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig index 34a2899e69c0..83f6ca4bf210 100644 --- a/drivers/tty/serial/Kconfig +++ b/drivers/tty/serial/Kconfig @@ -1295,14 +1295,6 @@ config SERIAL_AR933X_NR_UARTS Set this to the number of serial ports you want the driver to support. -config SERIAL_EFM32_UART - tristate "EFM32 UART/USART port" - depends on ARM && (ARCH_EFM32 || COMPILE_TEST) - select SERIAL_CORE - help - This driver support the USART and UART ports on - Energy Micro's efm32 SoCs. - config SERIAL_MPS2_UART_CONSOLE bool "MPS2 UART console support" depends on SERIAL_MPS2_UART @@ -1316,11 +1308,6 @@ config SERIAL_MPS2_UART help This driver support the UART ports on ARM MPS2. -config SERIAL_EFM32_UART_CONSOLE - bool "EFM32 UART/USART console support" - depends on SERIAL_EFM32_UART=y - select SERIAL_CORE_CONSOLE - config SERIAL_ARC tristate "ARC UART driver support" select SERIAL_CORE diff --git a/drivers/tty/serial/Makefile b/drivers/tty/serial/Makefile index b85d53f9e9ff..ec2b74091f0c 100644 --- a/drivers/tty/serial/Makefile +++ b/drivers/tty/serial/Makefile @@ -73,7 +73,6 @@ obj-$(CONFIG_SERIAL_SIRFSOC) += sirfsoc_uart.o obj-$(CONFIG_SERIAL_TEGRA) += serial-tegra.o obj-$(CONFIG_SERIAL_TEGRA_TCU) += tegra-tcu.o obj-$(CONFIG_SERIAL_AR933X) += ar933x_uart.o -obj-$(CONFIG_SERIAL_EFM32_UART) += efm32-uart.o obj-$(CONFIG_SERIAL_ARC) += arc_uart.o obj-$(CONFIG_SERIAL_RP2) += rp2.o obj-$(CONFIG_SERIAL_FSL_LPUART) += fsl_lpuart.o diff --git a/drivers/tty/serial/efm32-uart.c b/drivers/tty/serial/efm32-uart.c deleted file mode 100644 index f12f29cf4f31..000000000000 --- a/drivers/tty/serial/efm32-uart.c +++ /dev/null @@ -1,852 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#define DRIVER_NAME "efm32-uart" -#define DEV_NAME "ttyefm" - -#define UARTn_CTRL 0x00 -#define UARTn_CTRL_SYNC 0x0001 -#define UARTn_CTRL_TXBIL 0x1000 - -#define UARTn_FRAME 0x04 -#define UARTn_FRAME_DATABITS__MASK 0x000f -#define UARTn_FRAME_DATABITS(n) ((n) - 3) -#define UARTn_FRAME_PARITY__MASK 0x0300 -#define UARTn_FRAME_PARITY_NONE 0x0000 -#define UARTn_FRAME_PARITY_EVEN 0x0200 -#define UARTn_FRAME_PARITY_ODD 0x0300 -#define UARTn_FRAME_STOPBITS_HALF 0x0000 -#define UARTn_FRAME_STOPBITS_ONE 0x1000 -#define UARTn_FRAME_STOPBITS_TWO 0x3000 - -#define UARTn_CMD 0x0c -#define UARTn_CMD_RXEN 0x0001 -#define UARTn_CMD_RXDIS 0x0002 -#define UARTn_CMD_TXEN 0x0004 -#define UARTn_CMD_TXDIS 0x0008 - -#define UARTn_STATUS 0x10 -#define UARTn_STATUS_TXENS 0x0002 -#define UARTn_STATUS_TXC 0x0020 -#define UARTn_STATUS_TXBL 0x0040 -#define UARTn_STATUS_RXDATAV 0x0080 - -#define UARTn_CLKDIV 0x14 - -#define UARTn_RXDATAX 0x18 -#define UARTn_RXDATAX_RXDATA__MASK 0x01ff -#define UARTn_RXDATAX_PERR 0x4000 -#define UARTn_RXDATAX_FERR 0x8000 -/* - * This is a software only flag used for ignore_status_mask and - * read_status_mask! It's used for breaks that the hardware doesn't report - * explicitly. - */ -#define SW_UARTn_RXDATAX_BERR 0x2000 - -#define UARTn_TXDATA 0x34 - -#define UARTn_IF 0x40 -#define UARTn_IF_TXC 0x0001 -#define UARTn_IF_TXBL 0x0002 -#define UARTn_IF_RXDATAV 0x0004 -#define UARTn_IF_RXOF 0x0010 - -#define UARTn_IFS 0x44 -#define UARTn_IFC 0x48 -#define UARTn_IEN 0x4c - -#define UARTn_ROUTE 0x54 -#define UARTn_ROUTE_LOCATION__MASK 0x0700 -#define UARTn_ROUTE_LOCATION(n) (((n) << 8) & UARTn_ROUTE_LOCATION__MASK) -#define UARTn_ROUTE_RXPEN 0x0001 -#define UARTn_ROUTE_TXPEN 0x0002 - -struct efm32_uart_port { - struct uart_port port; - unsigned int txirq; - struct clk *clk; - struct efm32_uart_pdata pdata; -}; -#define to_efm_port(_port) container_of(_port, struct efm32_uart_port, port) -#define efm_debug(efm_port, format, arg...) \ - dev_dbg(efm_port->port.dev, format, ##arg) - -static void efm32_uart_write32(struct efm32_uart_port *efm_port, - u32 value, unsigned offset) -{ - writel_relaxed(value, efm_port->port.membase + offset); -} - -static u32 efm32_uart_read32(struct efm32_uart_port *efm_port, - unsigned offset) -{ - return readl_relaxed(efm_port->port.membase + offset); -} - -static unsigned int efm32_uart_tx_empty(struct uart_port *port) -{ - struct efm32_uart_port *efm_port = to_efm_port(port); - u32 status = efm32_uart_read32(efm_port, UARTn_STATUS); - - if (status & UARTn_STATUS_TXC) - return TIOCSER_TEMT; - else - return 0; -} - -static void efm32_uart_set_mctrl(struct uart_port *port, unsigned int mctrl) -{ - /* sorry, neither handshaking lines nor loop functionallity */ -} - -static unsigned int efm32_uart_get_mctrl(struct uart_port *port) -{ - /* sorry, no handshaking lines available */ - return TIOCM_CAR | TIOCM_CTS | TIOCM_DSR; -} - -static void efm32_uart_stop_tx(struct uart_port *port) -{ - struct efm32_uart_port *efm_port = to_efm_port(port); - u32 ien = efm32_uart_read32(efm_port, UARTn_IEN); - - efm32_uart_write32(efm_port, UARTn_CMD_TXDIS, UARTn_CMD); - ien &= ~(UARTn_IF_TXC | UARTn_IF_TXBL); - efm32_uart_write32(efm_port, ien, UARTn_IEN); -} - -static void efm32_uart_tx_chars(struct efm32_uart_port *efm_port) -{ - struct uart_port *port = &efm_port->port; - struct circ_buf *xmit = &port->state->xmit; - - while (efm32_uart_read32(efm_port, UARTn_STATUS) & - UARTn_STATUS_TXBL) { - if (port->x_char) { - port->icount.tx++; - efm32_uart_write32(efm_port, port->x_char, - UARTn_TXDATA); - port->x_char = 0; - continue; - } - if (!uart_circ_empty(xmit) && !uart_tx_stopped(port)) { - port->icount.tx++; - efm32_uart_write32(efm_port, xmit->buf[xmit->tail], - UARTn_TXDATA); - xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1); - } else - break; - } - - if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS) - uart_write_wakeup(port); - - if (!port->x_char && uart_circ_empty(xmit) && - efm32_uart_read32(efm_port, UARTn_STATUS) & - UARTn_STATUS_TXC) - efm32_uart_stop_tx(port); -} - -static void efm32_uart_start_tx(struct uart_port *port) -{ - struct efm32_uart_port *efm_port = to_efm_port(port); - u32 ien; - - efm32_uart_write32(efm_port, - UARTn_IF_TXBL | UARTn_IF_TXC, UARTn_IFC); - ien = efm32_uart_read32(efm_port, UARTn_IEN); - efm32_uart_write32(efm_port, - ien | UARTn_IF_TXBL | UARTn_IF_TXC, UARTn_IEN); - efm32_uart_write32(efm_port, UARTn_CMD_TXEN, UARTn_CMD); - - efm32_uart_tx_chars(efm_port); -} - -static void efm32_uart_stop_rx(struct uart_port *port) -{ - struct efm32_uart_port *efm_port = to_efm_port(port); - - efm32_uart_write32(efm_port, UARTn_CMD_RXDIS, UARTn_CMD); -} - -static void efm32_uart_break_ctl(struct uart_port *port, int ctl) -{ - /* not possible without fiddling with gpios */ -} - -static void efm32_uart_rx_chars(struct efm32_uart_port *efm_port) -{ - struct uart_port *port = &efm_port->port; - - while (efm32_uart_read32(efm_port, UARTn_STATUS) & - UARTn_STATUS_RXDATAV) { - u32 rxdata = efm32_uart_read32(efm_port, UARTn_RXDATAX); - int flag = 0; - - /* - * This is a reserved bit and I only saw it read as 0. But to be - * sure not to be confused too much by new devices adhere to the - * warning in the reference manual that reserved bits might - * read as 1 in the future. - */ - rxdata &= ~SW_UARTn_RXDATAX_BERR; - - port->icount.rx++; - - if ((rxdata & UARTn_RXDATAX_FERR) && - !(rxdata & UARTn_RXDATAX_RXDATA__MASK)) { - rxdata |= SW_UARTn_RXDATAX_BERR; - port->icount.brk++; - if (uart_handle_break(port)) - continue; - } else if (rxdata & UARTn_RXDATAX_PERR) - port->icount.parity++; - else if (rxdata & UARTn_RXDATAX_FERR) - port->icount.frame++; - - rxdata &= port->read_status_mask; - - if (rxdata & SW_UARTn_RXDATAX_BERR) - flag = TTY_BREAK; - else if (rxdata & UARTn_RXDATAX_PERR) - flag = TTY_PARITY; - else if (rxdata & UARTn_RXDATAX_FERR) - flag = TTY_FRAME; - else if (uart_handle_sysrq_char(port, - rxdata & UARTn_RXDATAX_RXDATA__MASK)) - continue; - - if ((rxdata & port->ignore_status_mask) == 0) - tty_insert_flip_char(&port->state->port, - rxdata & UARTn_RXDATAX_RXDATA__MASK, flag); - } -} - -static irqreturn_t efm32_uart_rxirq(int irq, void *data) -{ - struct efm32_uart_port *efm_port = data; - u32 irqflag = efm32_uart_read32(efm_port, UARTn_IF); - int handled = IRQ_NONE; - struct uart_port *port = &efm_port->port; - struct tty_port *tport = &port->state->port; - - spin_lock(&port->lock); - - if (irqflag & UARTn_IF_RXDATAV) { - efm32_uart_write32(efm_port, UARTn_IF_RXDATAV, UARTn_IFC); - efm32_uart_rx_chars(efm_port); - - handled = IRQ_HANDLED; - } - - if (irqflag & UARTn_IF_RXOF) { - efm32_uart_write32(efm_port, UARTn_IF_RXOF, UARTn_IFC); - port->icount.overrun++; - tty_insert_flip_char(tport, 0, TTY_OVERRUN); - - handled = IRQ_HANDLED; - } - - spin_unlock(&port->lock); - - tty_flip_buffer_push(tport); - - return handled; -} - -static irqreturn_t efm32_uart_txirq(int irq, void *data) -{ - struct efm32_uart_port *efm_port = data; - u32 irqflag = efm32_uart_read32(efm_port, UARTn_IF); - - /* TXBL doesn't need to be cleared */ - if (irqflag & UARTn_IF_TXC) - efm32_uart_write32(efm_port, UARTn_IF_TXC, UARTn_IFC); - - if (irqflag & (UARTn_IF_TXC | UARTn_IF_TXBL)) { - efm32_uart_tx_chars(efm_port); - return IRQ_HANDLED; - } else - return IRQ_NONE; -} - -static int efm32_uart_startup(struct uart_port *port) -{ - struct efm32_uart_port *efm_port = to_efm_port(port); - int ret; - - ret = clk_enable(efm_port->clk); - if (ret) { - efm_debug(efm_port, "failed to enable clk\n"); - goto err_clk_enable; - } - port->uartclk = clk_get_rate(efm_port->clk); - - /* Enable pins at configured location */ - efm32_uart_write32(efm_port, - UARTn_ROUTE_LOCATION(efm_port->pdata.location) | - UARTn_ROUTE_RXPEN | UARTn_ROUTE_TXPEN, - UARTn_ROUTE); - - ret = request_irq(port->irq, efm32_uart_rxirq, 0, - DRIVER_NAME, efm_port); - if (ret) { - efm_debug(efm_port, "failed to register rxirq\n"); - goto err_request_irq_rx; - } - - /* disable all irqs */ - efm32_uart_write32(efm_port, 0, UARTn_IEN); - - ret = request_irq(efm_port->txirq, efm32_uart_txirq, 0, - DRIVER_NAME, efm_port); - if (ret) { - efm_debug(efm_port, "failed to register txirq\n"); - free_irq(port->irq, efm_port); -err_request_irq_rx: - - clk_disable(efm_port->clk); - } else { - efm32_uart_write32(efm_port, - UARTn_IF_RXDATAV | UARTn_IF_RXOF, UARTn_IEN); - efm32_uart_write32(efm_port, UARTn_CMD_RXEN, UARTn_CMD); - } - -err_clk_enable: - return ret; -} - -static void efm32_uart_shutdown(struct uart_port *port) -{ - struct efm32_uart_port *efm_port = to_efm_port(port); - - efm32_uart_write32(efm_port, 0, UARTn_IEN); - free_irq(port->irq, efm_port); - - clk_disable(efm_port->clk); -} - -static void efm32_uart_set_termios(struct uart_port *port, - struct ktermios *new, struct ktermios *old) -{ - struct efm32_uart_port *efm_port = to_efm_port(port); - unsigned long flags; - unsigned baud; - u32 clkdiv; - u32 frame = 0; - - /* no modem control lines */ - new->c_cflag &= ~(CRTSCTS | CMSPAR); - - baud = uart_get_baud_rate(port, new, old, - DIV_ROUND_CLOSEST(port->uartclk, 16 * 8192), - DIV_ROUND_CLOSEST(port->uartclk, 16)); - - switch (new->c_cflag & CSIZE) { - case CS5: - frame |= UARTn_FRAME_DATABITS(5); - break; - case CS6: - frame |= UARTn_FRAME_DATABITS(6); - break; - case CS7: - frame |= UARTn_FRAME_DATABITS(7); - break; - case CS8: - frame |= UARTn_FRAME_DATABITS(8); - break; - } - - if (new->c_cflag & CSTOPB) - /* the receiver only verifies the first stop bit */ - frame |= UARTn_FRAME_STOPBITS_TWO; - else - frame |= UARTn_FRAME_STOPBITS_ONE; - - if (new->c_cflag & PARENB) { - if (new->c_cflag & PARODD) - frame |= UARTn_FRAME_PARITY_ODD; - else - frame |= UARTn_FRAME_PARITY_EVEN; - } else - frame |= UARTn_FRAME_PARITY_NONE; - - /* - * the 6 lowest bits of CLKDIV are dc, bit 6 has value 0.25. - * port->uartclk <= 14e6, so 4 * port->uartclk doesn't overflow. - */ - clkdiv = (DIV_ROUND_CLOSEST(4 * port->uartclk, 16 * baud) - 4) << 6; - - spin_lock_irqsave(&port->lock, flags); - - efm32_uart_write32(efm_port, - UARTn_CMD_TXDIS | UARTn_CMD_RXDIS, UARTn_CMD); - - port->read_status_mask = UARTn_RXDATAX_RXDATA__MASK; - if (new->c_iflag & INPCK) - port->read_status_mask |= - UARTn_RXDATAX_FERR | UARTn_RXDATAX_PERR; - if (new->c_iflag & (IGNBRK | BRKINT | PARMRK)) - port->read_status_mask |= SW_UARTn_RXDATAX_BERR; - - port->ignore_status_mask = 0; - if (new->c_iflag & IGNPAR) - port->ignore_status_mask |= - UARTn_RXDATAX_FERR | UARTn_RXDATAX_PERR; - if (new->c_iflag & IGNBRK) - port->ignore_status_mask |= SW_UARTn_RXDATAX_BERR; - - uart_update_timeout(port, new->c_cflag, baud); - - efm32_uart_write32(efm_port, UARTn_CTRL_TXBIL, UARTn_CTRL); - efm32_uart_write32(efm_port, frame, UARTn_FRAME); - efm32_uart_write32(efm_port, clkdiv, UARTn_CLKDIV); - - efm32_uart_write32(efm_port, UARTn_CMD_TXEN | UARTn_CMD_RXEN, - UARTn_CMD); - - spin_unlock_irqrestore(&port->lock, flags); -} - -static const char *efm32_uart_type(struct uart_port *port) -{ - return port->type == PORT_EFMUART ? "efm32-uart" : NULL; -} - -static void efm32_uart_release_port(struct uart_port *port) -{ - struct efm32_uart_port *efm_port = to_efm_port(port); - - clk_unprepare(efm_port->clk); - clk_put(efm_port->clk); - iounmap(port->membase); -} - -static int efm32_uart_request_port(struct uart_port *port) -{ - struct efm32_uart_port *efm_port = to_efm_port(port); - int ret; - - port->membase = ioremap(port->mapbase, 60); - if (!efm_port->port.membase) { - ret = -ENOMEM; - efm_debug(efm_port, "failed to remap\n"); - goto err_ioremap; - } - - efm_port->clk = clk_get(port->dev, NULL); - if (IS_ERR(efm_port->clk)) { - ret = PTR_ERR(efm_port->clk); - efm_debug(efm_port, "failed to get clock\n"); - goto err_clk_get; - } - - ret = clk_prepare(efm_port->clk); - if (ret) { - clk_put(efm_port->clk); -err_clk_get: - - iounmap(port->membase); -err_ioremap: - return ret; - } - return 0; -} - -static void efm32_uart_config_port(struct uart_port *port, int type) -{ - if (type & UART_CONFIG_TYPE && - !efm32_uart_request_port(port)) - port->type = PORT_EFMUART; -} - -static int efm32_uart_verify_port(struct uart_port *port, - struct serial_struct *serinfo) -{ - int ret = 0; - - if (serinfo->type != PORT_UNKNOWN && serinfo->type != PORT_EFMUART) - ret = -EINVAL; - - return ret; -} - -static const struct uart_ops efm32_uart_pops = { - .tx_empty = efm32_uart_tx_empty, - .set_mctrl = efm32_uart_set_mctrl, - .get_mctrl = efm32_uart_get_mctrl, - .stop_tx = efm32_uart_stop_tx, - .start_tx = efm32_uart_start_tx, - .stop_rx = efm32_uart_stop_rx, - .break_ctl = efm32_uart_break_ctl, - .startup = efm32_uart_startup, - .shutdown = efm32_uart_shutdown, - .set_termios = efm32_uart_set_termios, - .type = efm32_uart_type, - .release_port = efm32_uart_release_port, - .request_port = efm32_uart_request_port, - .config_port = efm32_uart_config_port, - .verify_port = efm32_uart_verify_port, -}; - -static struct efm32_uart_port *efm32_uart_ports[5]; - -#ifdef CONFIG_SERIAL_EFM32_UART_CONSOLE -static void efm32_uart_console_putchar(struct uart_port *port, int ch) -{ - struct efm32_uart_port *efm_port = to_efm_port(port); - unsigned int timeout = 0x400; - u32 status; - - while (1) { - status = efm32_uart_read32(efm_port, UARTn_STATUS); - - if (status & UARTn_STATUS_TXBL) - break; - if (!timeout--) - return; - } - efm32_uart_write32(efm_port, ch, UARTn_TXDATA); -} - -static void efm32_uart_console_write(struct console *co, const char *s, - unsigned int count) -{ - struct efm32_uart_port *efm_port = efm32_uart_ports[co->index]; - u32 status = efm32_uart_read32(efm_port, UARTn_STATUS); - unsigned int timeout = 0x400; - - if (!(status & UARTn_STATUS_TXENS)) - efm32_uart_write32(efm_port, UARTn_CMD_TXEN, UARTn_CMD); - - uart_console_write(&efm_port->port, s, count, - efm32_uart_console_putchar); - - /* Wait for the transmitter to become empty */ - while (1) { - u32 status = efm32_uart_read32(efm_port, UARTn_STATUS); - if (status & UARTn_STATUS_TXC) - break; - if (!timeout--) - break; - } - - if (!(status & UARTn_STATUS_TXENS)) - efm32_uart_write32(efm_port, UARTn_CMD_TXDIS, UARTn_CMD); -} - -static void efm32_uart_console_get_options(struct efm32_uart_port *efm_port, - int *baud, int *parity, int *bits) -{ - u32 ctrl = efm32_uart_read32(efm_port, UARTn_CTRL); - u32 route, clkdiv, frame; - - if (ctrl & UARTn_CTRL_SYNC) - /* not operating in async mode */ - return; - - route = efm32_uart_read32(efm_port, UARTn_ROUTE); - if (!(route & UARTn_ROUTE_TXPEN)) - /* tx pin not routed */ - return; - - clkdiv = efm32_uart_read32(efm_port, UARTn_CLKDIV); - - *baud = DIV_ROUND_CLOSEST(4 * efm_port->port.uartclk, - 16 * (4 + (clkdiv >> 6))); - - frame = efm32_uart_read32(efm_port, UARTn_FRAME); - switch (frame & UARTn_FRAME_PARITY__MASK) { - case UARTn_FRAME_PARITY_ODD: - *parity = 'o'; - break; - case UARTn_FRAME_PARITY_EVEN: - *parity = 'e'; - break; - default: - *parity = 'n'; - } - - *bits = (frame & UARTn_FRAME_DATABITS__MASK) - - UARTn_FRAME_DATABITS(4) + 4; - - efm_debug(efm_port, "get_opts: options=%d%c%d\n", - *baud, *parity, *bits); -} - -static int efm32_uart_console_setup(struct console *co, char *options) -{ - struct efm32_uart_port *efm_port; - int baud = 115200; - int bits = 8; - int parity = 'n'; - int flow = 'n'; - int ret; - - if (co->index < 0 || co->index >= ARRAY_SIZE(efm32_uart_ports)) { - unsigned i; - for (i = 0; i < ARRAY_SIZE(efm32_uart_ports); ++i) { - if (efm32_uart_ports[i]) { - pr_warn("efm32-console: fall back to console index %u (from %hhi)\n", - i, co->index); - co->index = i; - break; - } - } - } - - efm_port = efm32_uart_ports[co->index]; - if (!efm_port) { - pr_warn("efm32-console: No port at %d\n", co->index); - return -ENODEV; - } - - ret = clk_prepare(efm_port->clk); - if (ret) { - dev_warn(efm_port->port.dev, - "console: clk_prepare failed: %d\n", ret); - return ret; - } - - efm_port->port.uartclk = clk_get_rate(efm_port->clk); - - if (options) - uart_parse_options(options, &baud, &parity, &bits, &flow); - else - efm32_uart_console_get_options(efm_port, - &baud, &parity, &bits); - - return uart_set_options(&efm_port->port, co, baud, parity, bits, flow); -} - -static struct uart_driver efm32_uart_reg; - -static struct console efm32_uart_console = { - .name = DEV_NAME, - .write = efm32_uart_console_write, - .device = uart_console_device, - .setup = efm32_uart_console_setup, - .flags = CON_PRINTBUFFER, - .index = -1, - .data = &efm32_uart_reg, -}; - -#else -#define efm32_uart_console (*(struct console *)NULL) -#endif /* ifdef CONFIG_SERIAL_EFM32_UART_CONSOLE / else */ - -static struct uart_driver efm32_uart_reg = { - .owner = THIS_MODULE, - .driver_name = DRIVER_NAME, - .dev_name = DEV_NAME, - .nr = ARRAY_SIZE(efm32_uart_ports), - .cons = &efm32_uart_console, -}; - -static int efm32_uart_probe_dt(struct platform_device *pdev, - struct efm32_uart_port *efm_port) -{ - struct device_node *np = pdev->dev.of_node; - u32 location; - int ret; - - if (!np) - return 1; - - ret = of_property_read_u32(np, "energymicro,location", &location); - - if (ret) - /* fall back to wrongly namespaced property */ - ret = of_property_read_u32(np, "efm32,location", &location); - - if (ret) - /* fall back to old and (wrongly) generic property "location" */ - ret = of_property_read_u32(np, "location", &location); - - if (!ret) { - if (location > 5) { - dev_err(&pdev->dev, "invalid location\n"); - return -EINVAL; - } - efm_debug(efm_port, "using location %u\n", location); - efm_port->pdata.location = location; - } else { - efm_debug(efm_port, "fall back to location 0\n"); - } - - ret = of_alias_get_id(np, "serial"); - if (ret < 0) { - dev_err(&pdev->dev, "failed to get alias id: %d\n", ret); - return ret; - } else { - efm_port->port.line = ret; - return 0; - } - -} - -static int efm32_uart_probe(struct platform_device *pdev) -{ - struct efm32_uart_port *efm_port; - struct resource *res; - unsigned int line; - int ret; - - efm_port = kzalloc(sizeof(*efm_port), GFP_KERNEL); - if (!efm_port) { - dev_dbg(&pdev->dev, "failed to allocate private data\n"); - return -ENOMEM; - } - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) { - ret = -ENODEV; - dev_dbg(&pdev->dev, "failed to determine base address\n"); - goto err_get_base; - } - - if (resource_size(res) < 60) { - ret = -EINVAL; - dev_dbg(&pdev->dev, "memory resource too small\n"); - goto err_too_small; - } - - ret = platform_get_irq(pdev, 0); - if (ret <= 0) { - dev_dbg(&pdev->dev, "failed to get rx irq\n"); - goto err_get_rxirq; - } - - efm_port->port.irq = ret; - - ret = platform_get_irq(pdev, 1); - if (ret <= 0) - ret = efm_port->port.irq + 1; - - efm_port->txirq = ret; - - efm_port->port.dev = &pdev->dev; - efm_port->port.mapbase = res->start; - efm_port->port.type = PORT_EFMUART; - efm_port->port.iotype = UPIO_MEM32; - efm_port->port.fifosize = 2; - efm_port->port.has_sysrq = IS_ENABLED(CONFIG_SERIAL_EFM32_UART_CONSOLE); - efm_port->port.ops = &efm32_uart_pops; - efm_port->port.flags = UPF_BOOT_AUTOCONF; - - ret = efm32_uart_probe_dt(pdev, efm_port); - if (ret > 0) { - /* not created by device tree */ - const struct efm32_uart_pdata *pdata = dev_get_platdata(&pdev->dev); - - efm_port->port.line = pdev->id; - - if (pdata) - efm_port->pdata = *pdata; - } else if (ret < 0) - goto err_probe_dt; - - line = efm_port->port.line; - - if (line >= 0 && line < ARRAY_SIZE(efm32_uart_ports)) - efm32_uart_ports[line] = efm_port; - - ret = uart_add_one_port(&efm32_uart_reg, &efm_port->port); - if (ret) { - dev_dbg(&pdev->dev, "failed to add port: %d\n", ret); - - if (line >= 0 && line < ARRAY_SIZE(efm32_uart_ports)) - efm32_uart_ports[line] = NULL; -err_probe_dt: -err_get_rxirq: -err_too_small: -err_get_base: - kfree(efm_port); - } else { - platform_set_drvdata(pdev, efm_port); - dev_dbg(&pdev->dev, "\\o/\n"); - } - - return ret; -} - -static int efm32_uart_remove(struct platform_device *pdev) -{ - struct efm32_uart_port *efm_port = platform_get_drvdata(pdev); - unsigned int line = efm_port->port.line; - - uart_remove_one_port(&efm32_uart_reg, &efm_port->port); - - if (line >= 0 && line < ARRAY_SIZE(efm32_uart_ports)) - efm32_uart_ports[line] = NULL; - - kfree(efm_port); - - return 0; -} - -static const struct of_device_id efm32_uart_dt_ids[] = { - { - .compatible = "energymicro,efm32-uart", - }, { - /* doesn't follow the "vendor,device" scheme, don't use */ - .compatible = "efm32,uart", - }, { - /* sentinel */ - } -}; -MODULE_DEVICE_TABLE(of, efm32_uart_dt_ids); - -static struct platform_driver efm32_uart_driver = { - .probe = efm32_uart_probe, - .remove = efm32_uart_remove, - - .driver = { - .name = DRIVER_NAME, - .of_match_table = efm32_uart_dt_ids, - }, -}; - -static int __init efm32_uart_init(void) -{ - int ret; - - ret = uart_register_driver(&efm32_uart_reg); - if (ret) - return ret; - - ret = platform_driver_register(&efm32_uart_driver); - if (ret) - uart_unregister_driver(&efm32_uart_reg); - - pr_info("EFM32 UART/USART driver\n"); - - return ret; -} -module_init(efm32_uart_init); - -static void __exit efm32_uart_exit(void) -{ - platform_driver_unregister(&efm32_uart_driver); - uart_unregister_driver(&efm32_uart_reg); -} -module_exit(efm32_uart_exit); - -MODULE_AUTHOR("Uwe Kleine-Koenig "); -MODULE_DESCRIPTION("EFM32 UART/USART driver"); -MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("platform:" DRIVER_NAME); diff --git a/include/linux/platform_data/efm32-uart.h b/include/linux/platform_data/efm32-uart.h deleted file mode 100644 index ccbb8f11db75..000000000000 --- a/include/linux/platform_data/efm32-uart.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * - * - */ -#ifndef __LINUX_PLATFORM_DATA_EFM32_UART_H__ -#define __LINUX_PLATFORM_DATA_EFM32_UART_H__ - -#include - -/** - * struct efm32_uart_pdata - * @location: pinmux location for the I/O pins (to be written to the ROUTE - * register) - */ -struct efm32_uart_pdata { - u8 location; -}; -#endif /* ifndef __LINUX_PLATFORM_DATA_EFM32_UART_H__ */ diff --git a/include/uapi/linux/serial_core.h b/include/uapi/linux/serial_core.h index 62c22045fe65..c4042dcfdc0c 100644 --- a/include/uapi/linux/serial_core.h +++ b/include/uapi/linux/serial_core.h @@ -208,9 +208,6 @@ /* Atheros AR933X SoC */ #define PORT_AR933X 99 -/* Energy Micro efm32 SoC */ -#define PORT_EFMUART 100 - /* ARC (Synopsys) on-chip UART */ #define PORT_ARC 101 -- cgit v1.2.3 From 8d502ef682fdbe0ddf2274dae903b07baf1140e0 Mon Sep 17 00:00:00 2001 From: Craig Tatlor Date: Fri, 4 Dec 2020 05:54:56 +0300 Subject: fixp-arith: add a linear interpolation function Adds a function to interpolate against two points, this is carried arount as a helper function by tons of drivers. Signed-off-by: Craig Tatlor Signed-off-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20201204025509.1075506-3-dmitry.baryshkov@linaro.org Signed-off-by: Jonathan Cameron --- include/linux/fixp-arith.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fixp-arith.h b/include/linux/fixp-arith.h index 8396013785ef..281cb4f83dbe 100644 --- a/include/linux/fixp-arith.h +++ b/include/linux/fixp-arith.h @@ -141,4 +141,23 @@ static inline s32 fixp_sin32_rad(u32 radians, u32 twopi) #define fixp_cos32_rad(rad, twopi) \ fixp_sin32_rad(rad + twopi / 4, twopi) +/** + * fixp_linear_interpolate() - interpolates a value from two known points + * + * @x0: x value of point 0 + * @y0: y value of point 0 + * @x1: x value of point 1 + * @y1: y value of point 1 + * @x: the linear interpolant + */ +static inline int fixp_linear_interpolate(int x0, int y0, int x1, int y1, int x) +{ + if (y0 == y1 || x == x0) + return y0; + if (x1 == x0 || x == x1) + return y1; + + return y0 + ((y1 - y0) * (x - x0) / (x1 - x0)); +} + #endif -- cgit v1.2.3 From ec82edb258bbf0233306cc45f65d4b6092243ee9 Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Fri, 4 Dec 2020 05:54:59 +0300 Subject: iio: adc: move qcom-vadc-common.h to include dir qcom-vadc-common module will be used by ADC thermal monitoring driver, so move it to global include dir. Signed-off-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20201204025509.1075506-6-dmitry.baryshkov@linaro.org Signed-off-by: Jonathan Cameron --- drivers/iio/adc/qcom-pm8xxx-xoadc.c | 3 +- drivers/iio/adc/qcom-spmi-adc5.c | 2 +- drivers/iio/adc/qcom-spmi-vadc.c | 3 +- drivers/iio/adc/qcom-vadc-common.c | 3 +- drivers/iio/adc/qcom-vadc-common.h | 185 ------------------------------ include/linux/iio/adc/qcom-vadc-common.h | 187 +++++++++++++++++++++++++++++++ 6 files changed, 191 insertions(+), 192 deletions(-) delete mode 100644 drivers/iio/adc/qcom-vadc-common.h create mode 100644 include/linux/iio/adc/qcom-vadc-common.h (limited to 'include/linux') diff --git a/drivers/iio/adc/qcom-pm8xxx-xoadc.c b/drivers/iio/adc/qcom-pm8xxx-xoadc.c index 7e108da7d255..0610bf254771 100644 --- a/drivers/iio/adc/qcom-pm8xxx-xoadc.c +++ b/drivers/iio/adc/qcom-pm8xxx-xoadc.c @@ -10,6 +10,7 @@ * Author: Linus Walleij */ +#include #include #include #include @@ -21,8 +22,6 @@ #include #include -#include "qcom-vadc-common.h" - /* * Definitions for the "user processor" registers lifted from the v3.4 * Qualcomm tree. Their kernel has two out-of-tree drivers for the ADC: diff --git a/drivers/iio/adc/qcom-spmi-adc5.c b/drivers/iio/adc/qcom-spmi-adc5.c index c2da8f068b87..b10a0fcf09dc 100644 --- a/drivers/iio/adc/qcom-spmi-adc5.c +++ b/drivers/iio/adc/qcom-spmi-adc5.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -19,7 +20,6 @@ #include #include -#include "qcom-vadc-common.h" #define ADC5_USR_REVISION1 0x0 #define ADC5_USR_STATUS1 0x8 diff --git a/drivers/iio/adc/qcom-spmi-vadc.c b/drivers/iio/adc/qcom-spmi-vadc.c index b0388f8a69f4..05ff948372b3 100644 --- a/drivers/iio/adc/qcom-spmi-vadc.c +++ b/drivers/iio/adc/qcom-spmi-vadc.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -20,8 +21,6 @@ #include -#include "qcom-vadc-common.h" - /* VADC register and bit definitions */ #define VADC_REVISION2 0x1 #define VADC_REVISION2_SUPPORTED_VADC 1 diff --git a/drivers/iio/adc/qcom-vadc-common.c b/drivers/iio/adc/qcom-vadc-common.c index 40d77b3af1bb..ee94774b72e6 100644 --- a/drivers/iio/adc/qcom-vadc-common.c +++ b/drivers/iio/adc/qcom-vadc-common.c @@ -3,14 +3,13 @@ #include #include #include +#include #include #include #include #include #include -#include "qcom-vadc-common.h" - /* Voltage to temperature */ static const struct vadc_map_pt adcmap_100k_104ef_104fb[] = { {1758, -40}, diff --git a/drivers/iio/adc/qcom-vadc-common.h b/drivers/iio/adc/qcom-vadc-common.h deleted file mode 100644 index 7e5f6428e311..000000000000 --- a/drivers/iio/adc/qcom-vadc-common.h +++ /dev/null @@ -1,185 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Code shared between the different Qualcomm PMIC voltage ADCs - */ - -#ifndef QCOM_VADC_COMMON_H -#define QCOM_VADC_COMMON_H - -#define VADC_CONV_TIME_MIN_US 2000 -#define VADC_CONV_TIME_MAX_US 2100 - -/* Min ADC code represents 0V */ -#define VADC_MIN_ADC_CODE 0x6000 -/* Max ADC code represents full-scale range of 1.8V */ -#define VADC_MAX_ADC_CODE 0xa800 - -#define VADC_ABSOLUTE_RANGE_UV 625000 -#define VADC_RATIOMETRIC_RANGE 1800 - -#define VADC_DEF_PRESCALING 0 /* 1:1 */ -#define VADC_DEF_DECIMATION 0 /* 512 */ -#define VADC_DEF_HW_SETTLE_TIME 0 /* 0 us */ -#define VADC_DEF_AVG_SAMPLES 0 /* 1 sample */ -#define VADC_DEF_CALIB_TYPE VADC_CALIB_ABSOLUTE - -#define VADC_DECIMATION_MIN 512 -#define VADC_DECIMATION_MAX 4096 -#define ADC5_DEF_VBAT_PRESCALING 1 /* 1:3 */ -#define ADC5_DECIMATION_SHORT 250 -#define ADC5_DECIMATION_MEDIUM 420 -#define ADC5_DECIMATION_LONG 840 -/* Default decimation - 1024 for rev2, 840 for pmic5 */ -#define ADC5_DECIMATION_DEFAULT 2 -#define ADC5_DECIMATION_SAMPLES_MAX 3 - -#define VADC_HW_SETTLE_DELAY_MAX 10000 -#define VADC_HW_SETTLE_SAMPLES_MAX 16 -#define VADC_AVG_SAMPLES_MAX 512 -#define ADC5_AVG_SAMPLES_MAX 16 - -#define PMIC5_CHG_TEMP_SCALE_FACTOR 377500 -#define PMIC5_SMB_TEMP_CONSTANT 419400 -#define PMIC5_SMB_TEMP_SCALE_FACTOR 356 - -#define PMI_CHG_SCALE_1 -138890 -#define PMI_CHG_SCALE_2 391750000000LL - -#define VADC5_MAX_CODE 0x7fff -#define ADC5_FULL_SCALE_CODE 0x70e4 -#define ADC5_USR_DATA_CHECK 0x8000 - -#define R_PU_100K 100000 -#define RATIO_MAX_ADC7 BIT(14) - -#define DIE_TEMP_ADC7_SCALE_1 -60000 -#define DIE_TEMP_ADC7_SCALE_2 20000 -#define DIE_TEMP_ADC7_SCALE_FACTOR 1000 -#define DIE_TEMP_ADC7_MAX 160000 - -/** - * struct vadc_map_pt - Map the graph representation for ADC channel - * @x: Represent the ADC digitized code. - * @y: Represent the physical data which can be temperature, voltage, - * resistance. - */ -struct vadc_map_pt { - s32 x; - s32 y; -}; - -/* - * VADC_CALIB_ABSOLUTE: uses the 625mV and 1.25V as reference channels. - * VADC_CALIB_RATIOMETRIC: uses the reference voltage (1.8V) and GND for - * calibration. - */ -enum vadc_calibration { - VADC_CALIB_ABSOLUTE = 0, - VADC_CALIB_RATIOMETRIC -}; - -/** - * struct vadc_linear_graph - Represent ADC characteristics. - * @dy: numerator slope to calculate the gain. - * @dx: denominator slope to calculate the gain. - * @gnd: A/D word of the ground reference used for the channel. - * - * Each ADC device has different offset and gain parameters which are - * computed to calibrate the device. - */ -struct vadc_linear_graph { - s32 dy; - s32 dx; - s32 gnd; -}; - -/** - * struct vadc_prescale_ratio - Represent scaling ratio for ADC input. - * @num: the inverse numerator of the gain applied to the input channel. - * @den: the inverse denominator of the gain applied to the input channel. - */ -struct vadc_prescale_ratio { - u32 num; - u32 den; -}; - -/** - * enum vadc_scale_fn_type - Scaling function to convert ADC code to - * physical scaled units for the channel. - * SCALE_DEFAULT: Default scaling to convert raw adc code to voltage (uV). - * SCALE_THERM_100K_PULLUP: Returns temperature in millidegC. - * Uses a mapping table with 100K pullup. - * SCALE_PMIC_THERM: Returns result in milli degree's Centigrade. - * SCALE_XOTHERM: Returns XO thermistor voltage in millidegC. - * SCALE_PMI_CHG_TEMP: Conversion for PMI CHG temp - * SCALE_HW_CALIB_DEFAULT: Default scaling to convert raw adc code to - * voltage (uV) with hardware applied offset/slope values to adc code. - * SCALE_HW_CALIB_THERM_100K_PULLUP: Returns temperature in millidegC using - * lookup table. The hardware applies offset/slope to adc code. - * SCALE_HW_CALIB_XOTHERM: Returns XO thermistor voltage in millidegC using - * 100k pullup. The hardware applies offset/slope to adc code. - * SCALE_HW_CALIB_THERM_100K_PU_PM7: Returns temperature in millidegC using - * lookup table for PMIC7. The hardware applies offset/slope to adc code. - * SCALE_HW_CALIB_PMIC_THERM: Returns result in milli degree's Centigrade. - * The hardware applies offset/slope to adc code. - * SCALE_HW_CALIB_PMIC_THERM: Returns result in milli degree's Centigrade. - * The hardware applies offset/slope to adc code. This is for PMIC7. - * SCALE_HW_CALIB_PM5_CHG_TEMP: Returns result in millidegrees for PMIC5 - * charger temperature. - * SCALE_HW_CALIB_PM5_SMB_TEMP: Returns result in millidegrees for PMIC5 - * SMB1390 temperature. - */ -enum vadc_scale_fn_type { - SCALE_DEFAULT = 0, - SCALE_THERM_100K_PULLUP, - SCALE_PMIC_THERM, - SCALE_XOTHERM, - SCALE_PMI_CHG_TEMP, - SCALE_HW_CALIB_DEFAULT, - SCALE_HW_CALIB_THERM_100K_PULLUP, - SCALE_HW_CALIB_XOTHERM, - SCALE_HW_CALIB_THERM_100K_PU_PM7, - SCALE_HW_CALIB_PMIC_THERM, - SCALE_HW_CALIB_PMIC_THERM_PM7, - SCALE_HW_CALIB_PM5_CHG_TEMP, - SCALE_HW_CALIB_PM5_SMB_TEMP, - SCALE_HW_CALIB_INVALID, -}; - -struct adc5_data { - const u32 full_scale_code_volt; - const u32 full_scale_code_cur; - const struct adc5_channels *adc_chans; - const struct iio_info *info; - unsigned int *decimation; - unsigned int *hw_settle_1; - unsigned int *hw_settle_2; -}; - -int qcom_vadc_scale(enum vadc_scale_fn_type scaletype, - const struct vadc_linear_graph *calib_graph, - const struct vadc_prescale_ratio *prescale, - bool absolute, - u16 adc_code, int *result_mdec); - -struct qcom_adc5_scale_type { - int (*scale_fn)(const struct vadc_prescale_ratio *prescale, - const struct adc5_data *data, u16 adc_code, int *result); -}; - -int qcom_adc5_hw_scale(enum vadc_scale_fn_type scaletype, - unsigned int prescale_ratio, - const struct adc5_data *data, - u16 adc_code, int *result_mdec); - -int qcom_adc5_prescaling_from_dt(u32 num, u32 den); - -int qcom_adc5_hw_settle_time_from_dt(u32 value, const unsigned int *hw_settle); - -int qcom_adc5_avg_samples_from_dt(u32 value); - -int qcom_adc5_decimation_from_dt(u32 value, const unsigned int *decimation); - -int qcom_vadc_decimation_from_dt(u32 value); - -#endif /* QCOM_VADC_COMMON_H */ diff --git a/include/linux/iio/adc/qcom-vadc-common.h b/include/linux/iio/adc/qcom-vadc-common.h new file mode 100644 index 000000000000..03a9119edc71 --- /dev/null +++ b/include/linux/iio/adc/qcom-vadc-common.h @@ -0,0 +1,187 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Code shared between the different Qualcomm PMIC voltage ADCs + */ + +#ifndef QCOM_VADC_COMMON_H +#define QCOM_VADC_COMMON_H + +#include + +#define VADC_CONV_TIME_MIN_US 2000 +#define VADC_CONV_TIME_MAX_US 2100 + +/* Min ADC code represents 0V */ +#define VADC_MIN_ADC_CODE 0x6000 +/* Max ADC code represents full-scale range of 1.8V */ +#define VADC_MAX_ADC_CODE 0xa800 + +#define VADC_ABSOLUTE_RANGE_UV 625000 +#define VADC_RATIOMETRIC_RANGE 1800 + +#define VADC_DEF_PRESCALING 0 /* 1:1 */ +#define VADC_DEF_DECIMATION 0 /* 512 */ +#define VADC_DEF_HW_SETTLE_TIME 0 /* 0 us */ +#define VADC_DEF_AVG_SAMPLES 0 /* 1 sample */ +#define VADC_DEF_CALIB_TYPE VADC_CALIB_ABSOLUTE + +#define VADC_DECIMATION_MIN 512 +#define VADC_DECIMATION_MAX 4096 +#define ADC5_DEF_VBAT_PRESCALING 1 /* 1:3 */ +#define ADC5_DECIMATION_SHORT 250 +#define ADC5_DECIMATION_MEDIUM 420 +#define ADC5_DECIMATION_LONG 840 +/* Default decimation - 1024 for rev2, 840 for pmic5 */ +#define ADC5_DECIMATION_DEFAULT 2 +#define ADC5_DECIMATION_SAMPLES_MAX 3 + +#define VADC_HW_SETTLE_DELAY_MAX 10000 +#define VADC_HW_SETTLE_SAMPLES_MAX 16 +#define VADC_AVG_SAMPLES_MAX 512 +#define ADC5_AVG_SAMPLES_MAX 16 + +#define PMIC5_CHG_TEMP_SCALE_FACTOR 377500 +#define PMIC5_SMB_TEMP_CONSTANT 419400 +#define PMIC5_SMB_TEMP_SCALE_FACTOR 356 + +#define PMI_CHG_SCALE_1 -138890 +#define PMI_CHG_SCALE_2 391750000000LL + +#define VADC5_MAX_CODE 0x7fff +#define ADC5_FULL_SCALE_CODE 0x70e4 +#define ADC5_USR_DATA_CHECK 0x8000 + +#define R_PU_100K 100000 +#define RATIO_MAX_ADC7 BIT(14) + +#define DIE_TEMP_ADC7_SCALE_1 -60000 +#define DIE_TEMP_ADC7_SCALE_2 20000 +#define DIE_TEMP_ADC7_SCALE_FACTOR 1000 +#define DIE_TEMP_ADC7_MAX 160000 + +/** + * struct vadc_map_pt - Map the graph representation for ADC channel + * @x: Represent the ADC digitized code. + * @y: Represent the physical data which can be temperature, voltage, + * resistance. + */ +struct vadc_map_pt { + s32 x; + s32 y; +}; + +/* + * VADC_CALIB_ABSOLUTE: uses the 625mV and 1.25V as reference channels. + * VADC_CALIB_RATIOMETRIC: uses the reference voltage (1.8V) and GND for + * calibration. + */ +enum vadc_calibration { + VADC_CALIB_ABSOLUTE = 0, + VADC_CALIB_RATIOMETRIC +}; + +/** + * struct vadc_linear_graph - Represent ADC characteristics. + * @dy: numerator slope to calculate the gain. + * @dx: denominator slope to calculate the gain. + * @gnd: A/D word of the ground reference used for the channel. + * + * Each ADC device has different offset and gain parameters which are + * computed to calibrate the device. + */ +struct vadc_linear_graph { + s32 dy; + s32 dx; + s32 gnd; +}; + +/** + * struct vadc_prescale_ratio - Represent scaling ratio for ADC input. + * @num: the inverse numerator of the gain applied to the input channel. + * @den: the inverse denominator of the gain applied to the input channel. + */ +struct vadc_prescale_ratio { + u32 num; + u32 den; +}; + +/** + * enum vadc_scale_fn_type - Scaling function to convert ADC code to + * physical scaled units for the channel. + * SCALE_DEFAULT: Default scaling to convert raw adc code to voltage (uV). + * SCALE_THERM_100K_PULLUP: Returns temperature in millidegC. + * Uses a mapping table with 100K pullup. + * SCALE_PMIC_THERM: Returns result in milli degree's Centigrade. + * SCALE_XOTHERM: Returns XO thermistor voltage in millidegC. + * SCALE_PMI_CHG_TEMP: Conversion for PMI CHG temp + * SCALE_HW_CALIB_DEFAULT: Default scaling to convert raw adc code to + * voltage (uV) with hardware applied offset/slope values to adc code. + * SCALE_HW_CALIB_THERM_100K_PULLUP: Returns temperature in millidegC using + * lookup table. The hardware applies offset/slope to adc code. + * SCALE_HW_CALIB_XOTHERM: Returns XO thermistor voltage in millidegC using + * 100k pullup. The hardware applies offset/slope to adc code. + * SCALE_HW_CALIB_THERM_100K_PU_PM7: Returns temperature in millidegC using + * lookup table for PMIC7. The hardware applies offset/slope to adc code. + * SCALE_HW_CALIB_PMIC_THERM: Returns result in milli degree's Centigrade. + * The hardware applies offset/slope to adc code. + * SCALE_HW_CALIB_PMIC_THERM: Returns result in milli degree's Centigrade. + * The hardware applies offset/slope to adc code. This is for PMIC7. + * SCALE_HW_CALIB_PM5_CHG_TEMP: Returns result in millidegrees for PMIC5 + * charger temperature. + * SCALE_HW_CALIB_PM5_SMB_TEMP: Returns result in millidegrees for PMIC5 + * SMB1390 temperature. + */ +enum vadc_scale_fn_type { + SCALE_DEFAULT = 0, + SCALE_THERM_100K_PULLUP, + SCALE_PMIC_THERM, + SCALE_XOTHERM, + SCALE_PMI_CHG_TEMP, + SCALE_HW_CALIB_DEFAULT, + SCALE_HW_CALIB_THERM_100K_PULLUP, + SCALE_HW_CALIB_XOTHERM, + SCALE_HW_CALIB_THERM_100K_PU_PM7, + SCALE_HW_CALIB_PMIC_THERM, + SCALE_HW_CALIB_PMIC_THERM_PM7, + SCALE_HW_CALIB_PM5_CHG_TEMP, + SCALE_HW_CALIB_PM5_SMB_TEMP, + SCALE_HW_CALIB_INVALID, +}; + +struct adc5_data { + const u32 full_scale_code_volt; + const u32 full_scale_code_cur; + const struct adc5_channels *adc_chans; + const struct iio_info *info; + unsigned int *decimation; + unsigned int *hw_settle_1; + unsigned int *hw_settle_2; +}; + +int qcom_vadc_scale(enum vadc_scale_fn_type scaletype, + const struct vadc_linear_graph *calib_graph, + const struct vadc_prescale_ratio *prescale, + bool absolute, + u16 adc_code, int *result_mdec); + +struct qcom_adc5_scale_type { + int (*scale_fn)(const struct vadc_prescale_ratio *prescale, + const struct adc5_data *data, u16 adc_code, int *result); +}; + +int qcom_adc5_hw_scale(enum vadc_scale_fn_type scaletype, + unsigned int prescale_ratio, + const struct adc5_data *data, + u16 adc_code, int *result_mdec); + +int qcom_adc5_prescaling_from_dt(u32 num, u32 den); + +int qcom_adc5_hw_settle_time_from_dt(u32 value, const unsigned int *hw_settle); + +int qcom_adc5_avg_samples_from_dt(u32 value); + +int qcom_adc5_decimation_from_dt(u32 value, const unsigned int *decimation); + +int qcom_vadc_decimation_from_dt(u32 value); + +#endif /* QCOM_VADC_COMMON_H */ -- cgit v1.2.3 From 6e39b145cef7577cd6b550e633155e5fc1e5838e Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Fri, 4 Dec 2020 05:55:01 +0300 Subject: iio: provide of_iio_channel_get_by_name() and devm_ version it There might be cases when the IIO channel is attached to the device subnode instead of being attached to the main device node. Allow drivers to query IIO channels by using device tree nodes. Signed-off-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20201204025509.1075506-8-dmitry.baryshkov@linaro.org Signed-off-by: Jonathan Cameron --- drivers/iio/inkern.c | 34 ++++++++++++++++++++++++++-------- include/linux/iio/consumer.h | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/inkern.c b/drivers/iio/inkern.c index fe30bcb6a57b..db77a2d4a56b 100644 --- a/drivers/iio/inkern.c +++ b/drivers/iio/inkern.c @@ -191,8 +191,8 @@ err_free_channel: return ERR_PTR(err); } -static struct iio_channel *of_iio_channel_get_by_name(struct device_node *np, - const char *name) +struct iio_channel *of_iio_channel_get_by_name(struct device_node *np, + const char *name) { struct iio_channel *chan = NULL; @@ -230,6 +230,7 @@ static struct iio_channel *of_iio_channel_get_by_name(struct device_node *np, return chan; } +EXPORT_SYMBOL_GPL(of_iio_channel_get_by_name); static struct iio_channel *of_iio_channel_get_all(struct device *dev) { @@ -272,12 +273,6 @@ error_free_chans: #else /* CONFIG_OF */ -static inline struct iio_channel * -of_iio_channel_get_by_name(struct device_node *np, const char *name) -{ - return NULL; -} - static inline struct iio_channel *of_iio_channel_get_all(struct device *dev) { return NULL; @@ -393,6 +388,29 @@ struct iio_channel *devm_iio_channel_get(struct device *dev, } EXPORT_SYMBOL_GPL(devm_iio_channel_get); +struct iio_channel *devm_of_iio_channel_get_by_name(struct device *dev, + struct device_node *np, + const char *channel_name) +{ + struct iio_channel **ptr, *channel; + + ptr = devres_alloc(devm_iio_channel_free, sizeof(*ptr), GFP_KERNEL); + if (!ptr) + return ERR_PTR(-ENOMEM); + + channel = of_iio_channel_get_by_name(np, channel_name); + if (IS_ERR(channel)) { + devres_free(ptr); + return channel; + } + + *ptr = channel; + devres_add(dev, ptr); + + return channel; +} +EXPORT_SYMBOL_GPL(devm_of_iio_channel_get_by_name); + struct iio_channel *iio_channel_get_all(struct device *dev) { const char *name; diff --git a/include/linux/iio/consumer.h b/include/linux/iio/consumer.h index c4118dcb8e05..0a90ba8fa1bb 100644 --- a/include/linux/iio/consumer.h +++ b/include/linux/iio/consumer.h @@ -13,6 +13,7 @@ struct iio_dev; struct iio_chan_spec; struct device; +struct device_node; /** * struct iio_channel - everything needed for a consumer to use a channel @@ -97,6 +98,41 @@ void iio_channel_release_all(struct iio_channel *chan); */ struct iio_channel *devm_iio_channel_get_all(struct device *dev); +/** + * of_iio_channel_get_by_name() - get description of all that is needed to access channel. + * @np: Pointer to consumer device tree node + * @consumer_channel: Unique name to identify the channel on the consumer + * side. This typically describes the channels use within + * the consumer. E.g. 'battery_voltage' + */ +#ifdef CONFIG_OF +struct iio_channel *of_iio_channel_get_by_name(struct device_node *np, const char *name); +#else +static inline struct iio_channel * +of_iio_channel_get_by_name(struct device_node *np, const char *name) +{ + return NULL; +} +#endif + +/** + * devm_of_iio_channel_get_by_name() - Resource managed version of of_iio_channel_get_by_name(). + * @dev: Pointer to consumer device. + * @np: Pointer to consumer device tree node + * @consumer_channel: Unique name to identify the channel on the consumer + * side. This typically describes the channels use within + * the consumer. E.g. 'battery_voltage' + * + * Returns a pointer to negative errno if it is not able to get the iio channel + * otherwise returns valid pointer for iio channel. + * + * The allocated iio channel is automatically released when the device is + * unbound. + */ +struct iio_channel *devm_of_iio_channel_get_by_name(struct device *dev, + struct device_node *np, + const char *consumer_channel); + struct iio_cb_buffer; /** * iio_channel_get_all_cb() - register callback for triggered capture -- cgit v1.2.3 From bb01e263743293d37148f1fa00e3ea04dca416a5 Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Fri, 4 Dec 2020 05:55:02 +0300 Subject: iio: adc: move vadc_map_pt from header to the source file struct vadc_map_pt is not used outside of qcom-vadc-common.c, so move it there from the global header file. Signed-off-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20201204025509.1075506-9-dmitry.baryshkov@linaro.org Signed-off-by: Jonathan Cameron --- drivers/iio/adc/qcom-vadc-common.c | 11 +++++++++++ include/linux/iio/adc/qcom-vadc-common.h | 11 ----------- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/adc/qcom-vadc-common.c b/drivers/iio/adc/qcom-vadc-common.c index ee94774b72e6..45a38602f66a 100644 --- a/drivers/iio/adc/qcom-vadc-common.c +++ b/drivers/iio/adc/qcom-vadc-common.c @@ -10,6 +10,17 @@ #include #include +/** + * struct vadc_map_pt - Map the graph representation for ADC channel + * @x: Represent the ADC digitized code. + * @y: Represent the physical data which can be temperature, voltage, + * resistance. + */ +struct vadc_map_pt { + s32 x; + s32 y; +}; + /* Voltage to temperature */ static const struct vadc_map_pt adcmap_100k_104ef_104fb[] = { {1758, -40}, diff --git a/include/linux/iio/adc/qcom-vadc-common.h b/include/linux/iio/adc/qcom-vadc-common.h index 03a9119edc71..1d337dd9e3dc 100644 --- a/include/linux/iio/adc/qcom-vadc-common.h +++ b/include/linux/iio/adc/qcom-vadc-common.h @@ -59,17 +59,6 @@ #define DIE_TEMP_ADC7_SCALE_FACTOR 1000 #define DIE_TEMP_ADC7_MAX 160000 -/** - * struct vadc_map_pt - Map the graph representation for ADC channel - * @x: Represent the ADC digitized code. - * @y: Represent the physical data which can be temperature, voltage, - * resistance. - */ -struct vadc_map_pt { - s32 x; - s32 y; -}; - /* * VADC_CALIB_ABSOLUTE: uses the 625mV and 1.25V as reference channels. * VADC_CALIB_RATIOMETRIC: uses the reference voltage (1.8V) and GND for -- cgit v1.2.3 From 3bd0ceb566f700adc5d164f431d1da039374aa97 Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Fri, 4 Dec 2020 05:55:03 +0300 Subject: iio: adc: qcom-vadc-common: rewrite vadc7 die temp calculation qcom_vadc7_scale_hw_calib_die_temp() uses a table format different from the rest of volt/temp conversion functions in this file. Also the conversion functions results in non-monothonic values conversion, which seems wrong. Rewrite qcom_vadc7_scale_hw_calib_die_temp() to use qcom_vadc_map_voltage_temp() directly, like the rest of conversion functions do. Signed-off-by: Dmitry Baryshkov Link: https://lore.kernel.org/r/20201204025509.1075506-10-dmitry.baryshkov@linaro.org Signed-off-by: Jonathan Cameron --- drivers/iio/adc/qcom-vadc-common.c | 50 ++++++++++---------------------- include/linux/iio/adc/qcom-vadc-common.h | 5 ---- 2 files changed, 15 insertions(+), 40 deletions(-) (limited to 'include/linux') diff --git a/drivers/iio/adc/qcom-vadc-common.c b/drivers/iio/adc/qcom-vadc-common.c index 45a38602f66a..0c705bb473fe 100644 --- a/drivers/iio/adc/qcom-vadc-common.c +++ b/drivers/iio/adc/qcom-vadc-common.c @@ -101,18 +101,18 @@ static const struct vadc_map_pt adcmap_100k_104ef_104fb_1875_vref[] = { }; static const struct vadc_map_pt adcmap7_die_temp[] = { - { 433700, 1967}, - { 473100, 1964}, - { 512400, 1957}, - { 551500, 1949}, - { 590500, 1940}, - { 629300, 1930}, - { 667900, 1921}, - { 706400, 1910}, - { 744600, 1896}, - { 782500, 1878}, - { 820100, 1859}, - { 857300, 0}, + { 857300, 160000 }, + { 820100, 140000 }, + { 782500, 120000 }, + { 744600, 100000 }, + { 706400, 80000 }, + { 667900, 60000 }, + { 629300, 40000 }, + { 590500, 20000 }, + { 551500, 0 }, + { 512400, -20000 }, + { 473100, -40000 }, + { 433700, -60000 }, }; /* @@ -585,33 +585,13 @@ static int qcom_vadc7_scale_hw_calib_die_temp( u16 adc_code, int *result_mdec) { - int voltage, vtemp0, temp, i; + int voltage; voltage = qcom_vadc_scale_code_voltage_factor(adc_code, prescale, data, 1); - if (adcmap7_die_temp[0].x > voltage) { - *result_mdec = DIE_TEMP_ADC7_SCALE_1; - return 0; - } - - if (adcmap7_die_temp[ARRAY_SIZE(adcmap7_die_temp) - 1].x <= voltage) { - *result_mdec = DIE_TEMP_ADC7_MAX; - return 0; - } - - for (i = 0; i < ARRAY_SIZE(adcmap7_die_temp); i++) - if (adcmap7_die_temp[i].x > voltage) - break; - - vtemp0 = adcmap7_die_temp[i - 1].x; - voltage = voltage - vtemp0; - temp = div64_s64(voltage * DIE_TEMP_ADC7_SCALE_FACTOR, - adcmap7_die_temp[i - 1].y); - temp += DIE_TEMP_ADC7_SCALE_1 + (DIE_TEMP_ADC7_SCALE_2 * (i - 1)); - *result_mdec = temp; - - return 0; + return qcom_vadc_map_voltage_temp(adcmap7_die_temp, ARRAY_SIZE(adcmap7_die_temp), + voltage, result_mdec); } static int qcom_vadc_scale_hw_smb_temp( diff --git a/include/linux/iio/adc/qcom-vadc-common.h b/include/linux/iio/adc/qcom-vadc-common.h index 1d337dd9e3dc..58216124d89d 100644 --- a/include/linux/iio/adc/qcom-vadc-common.h +++ b/include/linux/iio/adc/qcom-vadc-common.h @@ -54,11 +54,6 @@ #define R_PU_100K 100000 #define RATIO_MAX_ADC7 BIT(14) -#define DIE_TEMP_ADC7_SCALE_1 -60000 -#define DIE_TEMP_ADC7_SCALE_2 20000 -#define DIE_TEMP_ADC7_SCALE_FACTOR 1000 -#define DIE_TEMP_ADC7_MAX 160000 - /* * VADC_CALIB_ABSOLUTE: uses the 625mV and 1.25V as reference channels. * VADC_CALIB_RATIOMETRIC: uses the reference voltage (1.8V) and GND for -- cgit v1.2.3 From 8ff059b8531f3b98e14f0461859fc7cdd95823e4 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 18 Jan 2021 13:38:42 +0100 Subject: efi: ia64: move IA64-only declarations to new asm/efi.h header Move some EFI related declarations that are only referenced on IA64 to a new asm/efi.h arch header. Cc: Tony Luck Cc: Fenghua Yu Signed-off-by: Ard Biesheuvel --- arch/ia64/include/asm/efi.h | 13 +++++++++++++ arch/ia64/kernel/efi.c | 1 + arch/ia64/kernel/machine_kexec.c | 1 + arch/ia64/kernel/mca.c | 1 + arch/ia64/kernel/smpboot.c | 1 + arch/ia64/kernel/time.c | 1 + arch/ia64/kernel/uncached.c | 4 +--- arch/ia64/mm/contig.c | 1 + arch/ia64/mm/discontig.c | 1 + arch/ia64/mm/init.c | 1 + include/linux/efi.h | 6 ------ 11 files changed, 22 insertions(+), 9 deletions(-) create mode 100644 arch/ia64/include/asm/efi.h (limited to 'include/linux') diff --git a/arch/ia64/include/asm/efi.h b/arch/ia64/include/asm/efi.h new file mode 100644 index 000000000000..6a4a50d8f19a --- /dev/null +++ b/arch/ia64/include/asm/efi.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_EFI_H +#define _ASM_EFI_H + +typedef int (*efi_freemem_callback_t) (u64 start, u64 end, void *arg); + +void *efi_get_pal_addr(void); +void efi_map_pal_code(void); +void efi_memmap_walk(efi_freemem_callback_t, void *); +void efi_memmap_walk_uc(efi_freemem_callback_t, void *); +void efi_gettimeofday(struct timespec64 *ts); + +#endif diff --git a/arch/ia64/kernel/efi.c b/arch/ia64/kernel/efi.c index f932b25fb817..dd7fd750bb93 100644 --- a/arch/ia64/kernel/efi.c +++ b/arch/ia64/kernel/efi.c @@ -34,6 +34,7 @@ #include #include +#include #include #include #include diff --git a/arch/ia64/kernel/machine_kexec.c b/arch/ia64/kernel/machine_kexec.c index efc9b568401c..af310dc8a356 100644 --- a/arch/ia64/kernel/machine_kexec.c +++ b/arch/ia64/kernel/machine_kexec.c @@ -16,6 +16,7 @@ #include #include +#include #include #include #include diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index 2703f7795672..0fea266b4d39 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -91,6 +91,7 @@ #include #include +#include #include #include #include diff --git a/arch/ia64/kernel/smpboot.c b/arch/ia64/kernel/smpboot.c index 093040f7e626..49b488580939 100644 --- a/arch/ia64/kernel/smpboot.c +++ b/arch/ia64/kernel/smpboot.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/ia64/kernel/time.c b/arch/ia64/kernel/time.c index ed9fc3d057a6..a37f161a66b1 100644 --- a/arch/ia64/kernel/time.c +++ b/arch/ia64/kernel/time.c @@ -26,6 +26,7 @@ #include #include +#include #include #include #include diff --git a/arch/ia64/kernel/uncached.c b/arch/ia64/kernel/uncached.c index 0750f367837d..51883a66aeb5 100644 --- a/arch/ia64/kernel/uncached.c +++ b/arch/ia64/kernel/uncached.c @@ -20,14 +20,12 @@ #include #include #include +#include #include #include #include #include - -extern void __init efi_memmap_walk_uc(efi_freemem_callback_t, void *); - struct uncached_pool { struct gen_pool *pool; struct mutex add_chunk_mutex; /* serialize adding a converted chunk */ diff --git a/arch/ia64/mm/contig.c b/arch/ia64/mm/contig.c index bfc4ecd0a2ab..62fe80a16f42 100644 --- a/arch/ia64/mm/contig.c +++ b/arch/ia64/mm/contig.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index c7311131156e..03b3a02375ff 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 9b5acf8fb092..24583a39fa1b 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -27,6 +27,7 @@ #include #include +#include #include #include #include diff --git a/include/linux/efi.h b/include/linux/efi.h index 763b816ba19c..0c31af36697c 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -167,8 +167,6 @@ struct capsule_info { int __efi_capsule_setup_info(struct capsule_info *cap_info); -typedef int (*efi_freemem_callback_t) (u64 start, u64 end, void *arg); - /* * Types and defines for Time Services */ @@ -605,10 +603,6 @@ efi_guid_to_str(efi_guid_t *guid, char *out) } extern void efi_init (void); -extern void *efi_get_pal_addr (void); -extern void efi_map_pal_code (void); -extern void efi_memmap_walk (efi_freemem_callback_t callback, void *arg); -extern void efi_gettimeofday (struct timespec64 *ts); #ifdef CONFIG_EFI extern void efi_enter_virtual_mode (void); /* switch EFI to virtual mode, if possible */ #else -- cgit v1.2.3 From f2fc9ff28d1c9bef7760516feadd38164044caae Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Wed, 13 Jan 2021 18:52:46 -0800 Subject: usb: ch9: Add USB 3.2 SSP attributes In preparation for USB 3.2 dual-lane support, add sublink speed attribute macros and enum usb_ssp_rate. A USB device that operates in SuperSpeed Plus may operate at different speed and lane count. These additional macros and enum values help specifying that. Signed-off-by: Thinh Nguyen Link: https://lore.kernel.org/r/ae9293ebd63a29f2a2035054753534d9eb123d74.1610592135.git.Thinh.Nguyen@synopsys.com Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/ch9.h | 9 +++++++++ include/uapi/linux/usb/ch9.h | 13 +++++++++++++ 2 files changed, 22 insertions(+) (limited to 'include/linux') diff --git a/include/linux/usb/ch9.h b/include/linux/usb/ch9.h index 604c6c514a50..86c50907634e 100644 --- a/include/linux/usb/ch9.h +++ b/include/linux/usb/ch9.h @@ -36,6 +36,15 @@ #include #include +/* USB 3.2 SuperSpeed Plus phy signaling rate generation and lane count */ + +enum usb_ssp_rate { + USB_SSP_GEN_UNKNOWN = 0, + USB_SSP_GEN_2x1, + USB_SSP_GEN_1x2, + USB_SSP_GEN_2x2, +}; + /** * usb_ep_type_string() - Returns human readable-name of the endpoint type. * @ep_type: The endpoint type to return human-readable name for. If it's not diff --git a/include/uapi/linux/usb/ch9.h b/include/uapi/linux/usb/ch9.h index 0f865ae4ba89..17ce56198c9a 100644 --- a/include/uapi/linux/usb/ch9.h +++ b/include/uapi/linux/usb/ch9.h @@ -968,9 +968,22 @@ struct usb_ssp_cap_descriptor { __le32 bmSublinkSpeedAttr[1]; /* list of sublink speed attrib entries */ #define USB_SSP_SUBLINK_SPEED_SSID (0xf) /* sublink speed ID */ #define USB_SSP_SUBLINK_SPEED_LSE (0x3 << 4) /* Lanespeed exponent */ +#define USB_SSP_SUBLINK_SPEED_LSE_BPS 0 +#define USB_SSP_SUBLINK_SPEED_LSE_KBPS 1 +#define USB_SSP_SUBLINK_SPEED_LSE_MBPS 2 +#define USB_SSP_SUBLINK_SPEED_LSE_GBPS 3 + #define USB_SSP_SUBLINK_SPEED_ST (0x3 << 6) /* Sublink type */ +#define USB_SSP_SUBLINK_SPEED_ST_SYM_RX 0 +#define USB_SSP_SUBLINK_SPEED_ST_ASYM_RX 1 +#define USB_SSP_SUBLINK_SPEED_ST_SYM_TX 2 +#define USB_SSP_SUBLINK_SPEED_ST_ASYM_TX 3 + #define USB_SSP_SUBLINK_SPEED_RSVD (0x3f << 8) /* Reserved */ #define USB_SSP_SUBLINK_SPEED_LP (0x3 << 14) /* Link protocol */ +#define USB_SSP_SUBLINK_SPEED_LP_SS 0 +#define USB_SSP_SUBLINK_SPEED_LP_SSP 1 + #define USB_SSP_SUBLINK_SPEED_LSM (0xff << 16) /* Lanespeed mantissa */ } __attribute__((packed)); -- cgit v1.2.3 From db615c6264cffcd8f117c5628db1794afbd0f254 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Wed, 13 Jan 2021 18:53:00 -0800 Subject: usb: gadget: Introduce SSP rates and lanes A USB device controller operating in SuperSpeed Plus may support gen2x1, gen1x2, and/or gen2x2. Introduce SuperSpeed Plus signaling rate generation and lane count to usb_gadget with the fields ssp_rate and max_ssp_rate. The gadget driver can use these to setup the device BOS descriptor and select the desire operating speed and number of lanes. Signed-off-by: Thinh Nguyen Link: https://lore.kernel.org/r/b6d2196dcc3c73747f91abf9a082b20bbe276cc4.1610592135.git.Thinh.Nguyen@synopsys.com Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/gadget.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include/linux') diff --git a/include/linux/usb/gadget.h b/include/linux/usb/gadget.h index e7351d64f11f..02483c862444 100644 --- a/include/linux/usb/gadget.h +++ b/include/linux/usb/gadget.h @@ -339,6 +339,10 @@ struct usb_gadget_ops { * @speed: Speed of current connection to USB host. * @max_speed: Maximal speed the UDC can handle. UDC must support this * and all slower speeds. + * @ssp_rate: Current connected SuperSpeed Plus signaling rate and lane count. + * @max_ssp_rate: Maximum SuperSpeed Plus signaling rate and lane count the UDC + * can handle. The UDC must support this and all slower speeds and lower + * number of lanes. * @state: the state we are now (attached, suspended, configured, etc) * @name: Identifies the controller hardware type. Used in diagnostics * and sometimes configuration. @@ -406,6 +410,11 @@ struct usb_gadget { struct list_head ep_list; /* of usb_ep */ enum usb_device_speed speed; enum usb_device_speed max_speed; + + /* USB SuperSpeed Plus only */ + enum usb_ssp_rate ssp_rate; + enum usb_ssp_rate max_ssp_rate; + enum usb_device_state state; const char *name; struct device dev; -- cgit v1.2.3 From ead4c124852e66b6aa033e34cf9c4f08d40aeffc Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Wed, 13 Jan 2021 18:53:07 -0800 Subject: usb: gadget: Introduce udc_set_ssp_rate() for SSP A SuperSpeed Plus device may operate at different speed and lane count (i.e. gen2x2, gen1x2, or gen2x1). Introduce gadget ops udc_set_ssp_rate() to set the desire corresponding usb_ssp_rate for SuperSpeed Plus capable devices. If the USB device supports different speeds at SuperSpeed Plus, set the device to operate with the maximum number of lanes and speed. Signed-off-by: Thinh Nguyen Link: https://lore.kernel.org/r/9b85357cdadc02e3f0d653fd05f89eb46af836e1.1610592135.git.Thinh.Nguyen@synopsys.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/gadget/udc/core.c | 16 +++++++++++----- include/linux/usb/gadget.h | 2 ++ 2 files changed, 13 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index 98cf9216f3cb..4173acdcd35b 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -1133,12 +1133,18 @@ static inline void usb_gadget_udc_stop(struct usb_udc *udc) static inline void usb_gadget_udc_set_speed(struct usb_udc *udc, enum usb_device_speed speed) { - if (udc->gadget->ops->udc_set_speed) { - enum usb_device_speed s; + struct usb_gadget *gadget = udc->gadget; + enum usb_device_speed s; - s = min(speed, udc->gadget->max_speed); - udc->gadget->ops->udc_set_speed(udc->gadget, s); - } + if (speed == USB_SPEED_UNKNOWN) + s = gadget->max_speed; + else + s = min(speed, gadget->max_speed); + + if (s == USB_SPEED_SUPER_PLUS && gadget->ops->udc_set_ssp_rate) + gadget->ops->udc_set_ssp_rate(gadget, gadget->max_ssp_rate); + else if (gadget->ops->udc_set_speed) + gadget->ops->udc_set_speed(gadget, s); } /** diff --git a/include/linux/usb/gadget.h b/include/linux/usb/gadget.h index 02483c862444..ee04ef214ce8 100644 --- a/include/linux/usb/gadget.h +++ b/include/linux/usb/gadget.h @@ -323,6 +323,8 @@ struct usb_gadget_ops { struct usb_gadget_driver *); int (*udc_stop)(struct usb_gadget *); void (*udc_set_speed)(struct usb_gadget *, enum usb_device_speed); + void (*udc_set_ssp_rate)(struct usb_gadget *gadget, + enum usb_ssp_rate rate); struct usb_ep *(*match_ep)(struct usb_gadget *, struct usb_endpoint_descriptor *, struct usb_ss_ep_comp_descriptor *); -- cgit v1.2.3 From 719a402cf60311b1cdff3f6320abaecdcc5e46b7 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Sun, 17 Jan 2021 16:59:42 +0200 Subject: net: netdevice: Add operation ndo_sk_get_lower_dev ndo_sk_get_lower_dev returns the lower netdev that corresponds to a given socket. Additionally, we implement a helper netdev_sk_get_lowest_dev() to get the lowest one in chain. Signed-off-by: Tariq Toukan Reviewed-by: Boris Pismenny Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 4 ++++ net/core/dev.c | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 5b949076ed23..02dcef4d66e2 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1398,6 +1398,8 @@ struct net_device_ops { struct net_device* (*ndo_get_xmit_slave)(struct net_device *dev, struct sk_buff *skb, bool all_slaves); + struct net_device* (*ndo_sk_get_lower_dev)(struct net_device *dev, + struct sock *sk); netdev_features_t (*ndo_fix_features)(struct net_device *dev, netdev_features_t features); int (*ndo_set_features)(struct net_device *dev, @@ -2858,6 +2860,8 @@ int init_dummy_netdev(struct net_device *dev); struct net_device *netdev_get_xmit_slave(struct net_device *dev, struct sk_buff *skb, bool all_slaves); +struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev, + struct sock *sk); struct net_device *dev_get_by_index(struct net *net, int ifindex); struct net_device *__dev_get_by_index(struct net *net, int ifindex); struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex); diff --git a/net/core/dev.c b/net/core/dev.c index bae35c1ae192..6b90520a01b1 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -8105,6 +8105,39 @@ struct net_device *netdev_get_xmit_slave(struct net_device *dev, } EXPORT_SYMBOL(netdev_get_xmit_slave); +static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev, + struct sock *sk) +{ + const struct net_device_ops *ops = dev->netdev_ops; + + if (!ops->ndo_sk_get_lower_dev) + return NULL; + return ops->ndo_sk_get_lower_dev(dev, sk); +} + +/** + * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket + * @dev: device + * @sk: the socket + * + * %NULL is returned if no lower device is found. + */ + +struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev, + struct sock *sk) +{ + struct net_device *lower; + + lower = netdev_sk_get_lower_dev(dev, sk); + while (lower) { + dev = lower; + lower = netdev_sk_get_lower_dev(dev, sk); + } + + return dev; +} +EXPORT_SYMBOL(netdev_sk_get_lowest_dev); + static void netdev_adjacent_add_links(struct net_device *dev) { struct netdev_adjacent *iter; -- cgit v1.2.3 From ab0da5a57188b80d16d3222236c4e184953a5bb4 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Wed, 30 Dec 2020 15:01:20 +0200 Subject: net/mlx5: Expose ifc bits for query modify header Expose ifc bits for query_modify_header_context_in to be used by DEVX. Signed-off-by: Yishai Hadas Signed-off-by: Leon Romanovsky --- include/linux/mlx5/mlx5_ifc.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 442c0160caab..cf692fc17f41 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -5904,6 +5904,18 @@ struct mlx5_ifc_dealloc_modify_header_context_in_bits { u8 reserved_at_60[0x20]; }; +struct mlx5_ifc_query_modify_header_context_in_bits { + u8 opcode[0x10]; + u8 uid[0x10]; + + u8 reserved_at_20[0x10]; + u8 op_mod[0x10]; + + u8 modify_header_id[0x20]; + + u8 reserved_at_60[0xa0]; +}; + struct mlx5_ifc_query_dct_out_bits { u8 status[0x8]; u8 reserved_at_8[0x18]; -- cgit v1.2.3 From 2f196059864fb0fe8f60c14a2cb214055b283e08 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 2 Nov 2020 17:11:49 +0100 Subject: efi/libstub: whitespace cleanup Trivial whitespace cleanup. Signed-off-by: Ard Biesheuvel --- include/linux/efi.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/efi.h b/include/linux/efi.h index 0c31af36697c..2537a246c2d6 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -29,10 +29,10 @@ #include #define EFI_SUCCESS 0 -#define EFI_LOAD_ERROR ( 1 | (1UL << (BITS_PER_LONG-1))) +#define EFI_LOAD_ERROR ( 1 | (1UL << (BITS_PER_LONG-1))) #define EFI_INVALID_PARAMETER ( 2 | (1UL << (BITS_PER_LONG-1))) #define EFI_UNSUPPORTED ( 3 | (1UL << (BITS_PER_LONG-1))) -#define EFI_BAD_BUFFER_SIZE ( 4 | (1UL << (BITS_PER_LONG-1))) +#define EFI_BAD_BUFFER_SIZE ( 4 | (1UL << (BITS_PER_LONG-1))) #define EFI_BUFFER_TOO_SMALL ( 5 | (1UL << (BITS_PER_LONG-1))) #define EFI_NOT_READY ( 6 | (1UL << (BITS_PER_LONG-1))) #define EFI_DEVICE_ERROR ( 7 | (1UL << (BITS_PER_LONG-1))) -- cgit v1.2.3 From 3820749ddcee694abfd5ae6cabc18aaab11eab34 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 2 Nov 2020 11:51:10 +0100 Subject: efi/libstub: move TPM related prototypes into efistub.h Move TPM related definitions that are only used in the EFI stub into efistub.h, which is a local header. Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/libstub/efistub.h | 9 +++++++++ include/linux/efi.h | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index 2b7438ba1fbc..cde0a2ef507d 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -849,4 +849,13 @@ void efi_handle_post_ebs_state(void); enum efi_secureboot_mode efi_get_secureboot(void); +#ifdef CONFIG_RESET_ATTACK_MITIGATION +void efi_enable_reset_attack_mitigation(void); +#else +static inline void +efi_enable_reset_attack_mitigation(void) { } +#endif + +void efi_retrieve_tpm2_eventlog(void); + #endif diff --git a/include/linux/efi.h b/include/linux/efi.h index 2537a246c2d6..8710f5710c1d 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1104,13 +1104,6 @@ enum efi_secureboot_mode efi_get_secureboot_mode(efi_get_variable_t *get_var) return efi_secureboot_mode_enabled; } -#ifdef CONFIG_RESET_ATTACK_MITIGATION -void efi_enable_reset_attack_mitigation(void); -#else -static inline void -efi_enable_reset_attack_mitigation(void) { } -#endif - #ifdef CONFIG_EFI_EMBEDDED_FIRMWARE void efi_check_for_embedded_firmwares(void); #else @@ -1119,8 +1112,6 @@ static inline void efi_check_for_embedded_firmwares(void) { } efi_status_t efi_random_get_seed(void); -void efi_retrieve_tpm2_eventlog(void); - /* * Arch code can implement the following three template macros, avoiding * reptition for the void/non-void return cases of {__,}efi_call_virt(): -- cgit v1.2.3 From 7eab14de73a8028f770e703962c5437a2b0dda82 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Sat, 16 Jan 2021 16:13:22 +0000 Subject: mdio, phy: fix -Wshadow warnings triggered by nested container_of() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit container_of() macro hides a local variable '__mptr' inside. This becomes a problem when several container_of() are nested in each other within single line or plain macros. As C preprocessor doesn't support generating random variable names, the sole solution is to avoid defining macros that consist only of container_of() calls, or they will self-shadow '__mptr' each time: In file included from ./include/linux/bitmap.h:10, from drivers/net/phy/phy_device.c:12: drivers/net/phy/phy_device.c: In function ‘phy_device_release’: ./include/linux/kernel.h:693:8: warning: declaration of ‘__mptr’ shadows a previous local [-Wshadow] 693 | void *__mptr = (void *)(ptr); \ | ^~~~~~ ./include/linux/phy.h:647:26: note: in expansion of macro ‘container_of’ 647 | #define to_phy_device(d) container_of(to_mdio_device(d), \ | ^~~~~~~~~~~~ ./include/linux/mdio.h:52:27: note: in expansion of macro ‘container_of’ 52 | #define to_mdio_device(d) container_of(d, struct mdio_device, dev) | ^~~~~~~~~~~~ ./include/linux/phy.h:647:39: note: in expansion of macro ‘to_mdio_device’ 647 | #define to_phy_device(d) container_of(to_mdio_device(d), \ | ^~~~~~~~~~~~~~ drivers/net/phy/phy_device.c:217:8: note: in expansion of macro ‘to_phy_device’ 217 | kfree(to_phy_device(dev)); | ^~~~~~~~~~~~~ ./include/linux/kernel.h:693:8: note: shadowed declaration is here 693 | void *__mptr = (void *)(ptr); \ | ^~~~~~ ./include/linux/phy.h:647:26: note: in expansion of macro ‘container_of’ 647 | #define to_phy_device(d) container_of(to_mdio_device(d), \ | ^~~~~~~~~~~~ drivers/net/phy/phy_device.c:217:8: note: in expansion of macro ‘to_phy_device’ 217 | kfree(to_phy_device(dev)); | ^~~~~~~~~~~~~ As they are declared in header files, these warnings are highly repetitive and very annoying (along with the one from linux/pci.h). Convert the related macros from linux/{mdio,phy}.h to static inlines to avoid self-shadowing and potentially improve bug-catching. No functional changes implied. Signed-off-by: Alexander Lobakin Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/r/20210116161246.67075-1-alobakin@pm.me Signed-off-by: Jakub Kicinski --- include/linux/mdio.h | 23 ++++++++++++++++++----- include/linux/phy.h | 7 +++++-- 2 files changed, 23 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mdio.h b/include/linux/mdio.h index dbd69b3d170b..ffb787d5ebde 100644 --- a/include/linux/mdio.h +++ b/include/linux/mdio.h @@ -49,7 +49,11 @@ struct mdio_device { unsigned int reset_assert_delay; unsigned int reset_deassert_delay; }; -#define to_mdio_device(d) container_of(d, struct mdio_device, dev) + +static inline struct mdio_device *to_mdio_device(const struct device *dev) +{ + return container_of(dev, struct mdio_device, dev); +} /* struct mdio_driver_common: Common to all MDIO drivers */ struct mdio_driver_common { @@ -57,8 +61,12 @@ struct mdio_driver_common { int flags; }; #define MDIO_DEVICE_FLAG_PHY 1 -#define to_mdio_common_driver(d) \ - container_of(d, struct mdio_driver_common, driver) + +static inline struct mdio_driver_common * +to_mdio_common_driver(const struct device_driver *driver) +{ + return container_of(driver, struct mdio_driver_common, driver); +} /* struct mdio_driver: Generic MDIO driver */ struct mdio_driver { @@ -73,8 +81,13 @@ struct mdio_driver { /* Clears up any memory if needed */ void (*remove)(struct mdio_device *mdiodev); }; -#define to_mdio_driver(d) \ - container_of(to_mdio_common_driver(d), struct mdio_driver, mdiodrv) + +static inline struct mdio_driver * +to_mdio_driver(const struct device_driver *driver) +{ + return container_of(to_mdio_common_driver(driver), struct mdio_driver, + mdiodrv); +} /* device driver data */ static inline void mdiodev_set_drvdata(struct mdio_device *mdio, void *data) diff --git a/include/linux/phy.h b/include/linux/phy.h index 24fcc6456a9e..bc323fbdd21e 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -648,8 +648,11 @@ struct phy_device { const struct macsec_ops *macsec_ops; #endif }; -#define to_phy_device(d) container_of(to_mdio_device(d), \ - struct phy_device, mdio) + +static inline struct phy_device *to_phy_device(const struct device *dev) +{ + return container_of(to_mdio_device(dev), struct phy_device, mdio); +} /** * struct phy_tdr_config - Configuration of a TDR raw test -- cgit v1.2.3 From fa82117010430aff2ce86400f7328f55a31b48a6 Mon Sep 17 00:00:00 2001 From: Xin Long Date: Sat, 16 Jan 2021 14:13:37 +0800 Subject: net: add inline function skb_csum_is_sctp This patch is to define a inline function skb_csum_is_sctp(), and also replace all places where it checks if it's a SCTP CSUM skb. This function would be used later in many networking drivers in the following patches. Suggested-by: Alexander Duyck Signed-off-by: Xin Long Reviewed-by: Alexander Duyck Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/pensando/ionic/ionic_txrx.c | 2 +- include/linux/skbuff.h | 5 +++++ net/core/dev.c | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c index ac4cd5d82e69..162a1ff1e9d2 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c @@ -979,7 +979,7 @@ static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb) stats->vlan_inserted++; } - if (skb->csum_not_inet) + if (skb_csum_is_sctp(skb)) stats->crc32_csum++; else stats->csum++; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index c9568cf60c2a..46f901adf1a8 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -4621,6 +4621,11 @@ static inline void skb_reset_redirect(struct sk_buff *skb) #endif } +static inline bool skb_csum_is_sctp(struct sk_buff *skb) +{ + return skb->csum_not_inet; +} + static inline void skb_set_kcov_handle(struct sk_buff *skb, const u64 kcov_handle) { diff --git a/net/core/dev.c b/net/core/dev.c index 6b90520a01b1..0332f2e8f7da 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3617,7 +3617,7 @@ static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, int skb_csum_hwoffload_help(struct sk_buff *skb, const netdev_features_t features) { - if (unlikely(skb->csum_not_inet)) + if (unlikely(skb_csum_is_sctp(skb))) return !!(features & NETIF_F_SCTP_CRC) ? 0 : skb_crc32c_csum_help(skb); -- cgit v1.2.3 From 7b8fc0103bb51d1d3e1fb5fd67958612e709f883 Mon Sep 17 00:00:00 2001 From: Jarod Wilson Date: Mon, 18 Jan 2021 20:09:27 -0500 Subject: bonding: add a vlan+srcmac tx hashing option This comes from an end-user request, where they're running multiple VMs on hosts with bonded interfaces connected to some interest switch topologies, where 802.3ad isn't an option. They're currently running a proprietary solution that effectively achieves load-balancing of VMs and bandwidth utilization improvements with a similar form of transmission algorithm. Basically, each VM has it's own vlan, so it always sends its traffic out the same interface, unless that interface fails. Traffic gets split between the interfaces, maintaining a consistent path, with failover still available if an interface goes down. Unlike bond_eth_hash(), this hash function is using the full source MAC address instead of just the last byte, as there are so few components to the hash, and in the no-vlan case, we would be returning just the last byte of the source MAC as the hash value. It's entirely possible to have two NICs in a bond with the same last byte of their MAC, but not the same MAC, so this adjustment should guarantee distinct hashes in all cases. This has been rudimetarily tested to provide similar results to the proprietary solution it is aiming to replace. A patch for iproute2 is also posted, to properly support the new mode there as well. Cc: Jay Vosburgh Cc: Veaceslav Falico Cc: Andy Gospodarek Cc: Thomas Davis Signed-off-by: Jarod Wilson Link: https://lore.kernel.org/r/20210119010927.1191922-1-jarod@redhat.com Signed-off-by: Jakub Kicinski --- Documentation/networking/bonding.rst | 13 +++++++++++++ drivers/net/bonding/bond_main.c | 34 ++++++++++++++++++++++++++++++++-- drivers/net/bonding/bond_options.c | 13 +++++++------ include/linux/netdevice.h | 1 + include/uapi/linux/if_bonding.h | 1 + 5 files changed, 54 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/bonding.rst b/Documentation/networking/bonding.rst index adc314639085..5f690f0ad0e4 100644 --- a/Documentation/networking/bonding.rst +++ b/Documentation/networking/bonding.rst @@ -951,6 +951,19 @@ xmit_hash_policy packets will be distributed according to the encapsulated flows. + vlan+srcmac + + This policy uses a very rudimentary vlan ID and source mac + hash to load-balance traffic per-vlan, with failover + should one leg fail. The intended use case is for a bond + shared by multiple virtual machines, all configured to + use their own vlan, to give lacp-like functionality + without requiring lacp-capable switching hardware. + + The formula for the hash is simply + + hash = (vlan ID) XOR (source MAC vendor) XOR (source MAC dev) + The default value is layer2. This option was added in bonding version 2.6.3. In earlier versions of bonding, this parameter does not exist, and the layer2 policy is the only policy. The diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 539c6bc218df..74cbbb22470b 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -167,7 +167,7 @@ module_param(xmit_hash_policy, charp, 0); MODULE_PARM_DESC(xmit_hash_policy, "balance-alb, balance-tlb, balance-xor, 802.3ad hashing method; " "0 for layer 2 (default), 1 for layer 3+4, " "2 for layer 2+3, 3 for encap layer 2+3, " - "4 for encap layer 3+4"); + "4 for encap layer 3+4, 5 for vlan+srcmac"); module_param(arp_interval, int, 0); MODULE_PARM_DESC(arp_interval, "arp interval in milliseconds"); module_param_array(arp_ip_target, charp, NULL, 0); @@ -1457,6 +1457,8 @@ static enum netdev_lag_hash bond_lag_hash_type(struct bonding *bond, return NETDEV_LAG_HASH_E23; case BOND_XMIT_POLICY_ENCAP34: return NETDEV_LAG_HASH_E34; + case BOND_XMIT_POLICY_VLAN_SRCMAC: + return NETDEV_LAG_HASH_VLAN_SRCMAC; default: return NETDEV_LAG_HASH_UNKNOWN; } @@ -3519,6 +3521,27 @@ static bool bond_flow_ip(struct sk_buff *skb, struct flow_keys *fk, return true; } +static u32 bond_vlan_srcmac_hash(struct sk_buff *skb) +{ + struct ethhdr *mac_hdr = (struct ethhdr *)skb_mac_header(skb); + u32 srcmac_vendor = 0, srcmac_dev = 0; + u16 vlan; + int i; + + for (i = 0; i < 3; i++) + srcmac_vendor = (srcmac_vendor << 8) | mac_hdr->h_source[i]; + + for (i = 3; i < ETH_ALEN; i++) + srcmac_dev = (srcmac_dev << 8) | mac_hdr->h_source[i]; + + if (!skb_vlan_tag_present(skb)) + return srcmac_vendor ^ srcmac_dev; + + vlan = skb_vlan_tag_get(skb); + + return vlan ^ srcmac_vendor ^ srcmac_dev; +} + /* Extract the appropriate headers based on bond's xmit policy */ static bool bond_flow_dissect(struct bonding *bond, struct sk_buff *skb, struct flow_keys *fk) @@ -3526,10 +3549,14 @@ static bool bond_flow_dissect(struct bonding *bond, struct sk_buff *skb, bool l34 = bond->params.xmit_policy == BOND_XMIT_POLICY_LAYER34; int noff, proto = -1; - if (bond->params.xmit_policy > BOND_XMIT_POLICY_LAYER23) { + switch (bond->params.xmit_policy) { + case BOND_XMIT_POLICY_ENCAP23: + case BOND_XMIT_POLICY_ENCAP34: memset(fk, 0, sizeof(*fk)); return __skb_flow_dissect(NULL, skb, &flow_keys_bonding, fk, NULL, 0, 0, 0, 0); + default: + break; } fk->ports.ports = 0; @@ -3591,6 +3618,9 @@ u32 bond_xmit_hash(struct bonding *bond, struct sk_buff *skb) skb->l4_hash) return skb->hash; + if (bond->params.xmit_policy == BOND_XMIT_POLICY_VLAN_SRCMAC) + return bond_vlan_srcmac_hash(skb); + if (bond->params.xmit_policy == BOND_XMIT_POLICY_LAYER2 || !bond_flow_dissect(bond, skb, &flow)) return bond_eth_hash(skb); diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c index 8fcbf7f9c7b2..77d7c38bd435 100644 --- a/drivers/net/bonding/bond_options.c +++ b/drivers/net/bonding/bond_options.c @@ -96,12 +96,13 @@ static const struct bond_opt_value bond_pps_tbl[] = { }; static const struct bond_opt_value bond_xmit_hashtype_tbl[] = { - { "layer2", BOND_XMIT_POLICY_LAYER2, BOND_VALFLAG_DEFAULT}, - { "layer3+4", BOND_XMIT_POLICY_LAYER34, 0}, - { "layer2+3", BOND_XMIT_POLICY_LAYER23, 0}, - { "encap2+3", BOND_XMIT_POLICY_ENCAP23, 0}, - { "encap3+4", BOND_XMIT_POLICY_ENCAP34, 0}, - { NULL, -1, 0}, + { "layer2", BOND_XMIT_POLICY_LAYER2, BOND_VALFLAG_DEFAULT}, + { "layer3+4", BOND_XMIT_POLICY_LAYER34, 0}, + { "layer2+3", BOND_XMIT_POLICY_LAYER23, 0}, + { "encap2+3", BOND_XMIT_POLICY_ENCAP23, 0}, + { "encap3+4", BOND_XMIT_POLICY_ENCAP34, 0}, + { "vlan+srcmac", BOND_XMIT_POLICY_VLAN_SRCMAC, 0}, + { NULL, -1, 0}, }; static const struct bond_opt_value bond_arp_validate_tbl[] = { diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 02dcef4d66e2..ef517254367d 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2617,6 +2617,7 @@ enum netdev_lag_hash { NETDEV_LAG_HASH_L23, NETDEV_LAG_HASH_E23, NETDEV_LAG_HASH_E34, + NETDEV_LAG_HASH_VLAN_SRCMAC, NETDEV_LAG_HASH_UNKNOWN, }; diff --git a/include/uapi/linux/if_bonding.h b/include/uapi/linux/if_bonding.h index 45f3750aa861..e8eb4ad03cf1 100644 --- a/include/uapi/linux/if_bonding.h +++ b/include/uapi/linux/if_bonding.h @@ -94,6 +94,7 @@ #define BOND_XMIT_POLICY_LAYER23 2 /* layer 2+3 (IP ^ MAC) */ #define BOND_XMIT_POLICY_ENCAP23 3 /* encapsulated layer 2+3 */ #define BOND_XMIT_POLICY_ENCAP34 4 /* encapsulated layer 3+4 */ +#define BOND_XMIT_POLICY_VLAN_SRCMAC 5 /* vlan + source MAC */ /* 802.3ad port state definitions (43.4.2.2 in the 802.3ad standard) */ #define LACP_STATE_LACP_ACTIVITY 0x1 -- cgit v1.2.3 From 67c6bb56b649590a3f59c2a92331aa4e83d4534c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 6 Jan 2021 10:34:49 +0000 Subject: firmware: smccc: Add SMCCC TRNG function call IDs The ARM architected TRNG firmware interface, described in ARM spec DEN0098, define an ARM SMCCC based interface to a true random number generator, provided by firmware. Add the definitions of the SMCCC functions as defined by the spec. Signed-off-by: Ard Biesheuvel Signed-off-by: Andre Przywara Reviewed-by: Linus Walleij Reviewed-by: Sudeep Holla Link: https://lore.kernel.org/r/20210106103453.152275-2-andre.przywara@arm.com Signed-off-by: Will Deacon --- include/linux/arm-smccc.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'include/linux') diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h index f860645f6512..62c54234576c 100644 --- a/include/linux/arm-smccc.h +++ b/include/linux/arm-smccc.h @@ -102,6 +102,37 @@ ARM_SMCCC_OWNER_STANDARD_HYP, \ 0x21) +/* TRNG entropy source calls (defined by ARM DEN0098) */ +#define ARM_SMCCC_TRNG_VERSION \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_STANDARD, \ + 0x50) + +#define ARM_SMCCC_TRNG_FEATURES \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_STANDARD, \ + 0x51) + +#define ARM_SMCCC_TRNG_GET_UUID \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_STANDARD, \ + 0x52) + +#define ARM_SMCCC_TRNG_RND32 \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_STANDARD, \ + 0x53) + +#define ARM_SMCCC_TRNG_RND64 \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_STANDARD, \ + 0x53) + /* * Return codes defined in ARM DEN 0070A * ARM DEN 0070A is now merged/consolidated into ARM DEN 0028 C -- cgit v1.2.3 From f9ce0be71d1fbb038ada15ced83474b0e63f264d Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Sat, 19 Dec 2020 15:19:23 +0300 Subject: mm: Cleanup faultaround and finish_fault() codepaths alloc_set_pte() has two users with different requirements: in the faultaround code, it called from an atomic context and PTE page table has to be preallocated. finish_fault() can sleep and allocate page table as needed. PTL locking rules are also strange, hard to follow and overkill for finish_fault(). Let's untangle the mess. alloc_set_pte() has gone now. All locking is explicit. The price is some code duplication to handle huge pages in faultaround path, but it should be fine, having overall improvement in readability. Link: https://lore.kernel.org/r/20201229132819.najtavneutnf7ajp@box Signed-off-by: Kirill A. Shutemov [will: s/from from/from/ in comment; spotted by willy] Signed-off-by: Will Deacon --- fs/xfs/xfs_file.c | 6 +- include/linux/mm.h | 12 +-- include/linux/pgtable.h | 11 +++ mm/filemap.c | 177 +++++++++++++++++++++++++++++++----------- mm/memory.c | 199 ++++++++++++++---------------------------------- 5 files changed, 213 insertions(+), 192 deletions(-) (limited to 'include/linux') diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 5b0f93f73837..111fe73bb8a7 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -1319,17 +1319,19 @@ xfs_filemap_pfn_mkwrite( return __xfs_filemap_fault(vmf, PE_SIZE_PTE, true); } -static void +static vm_fault_t xfs_filemap_map_pages( struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff) { struct inode *inode = file_inode(vmf->vma->vm_file); + vm_fault_t ret; xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED); - filemap_map_pages(vmf, start_pgoff, end_pgoff); + ret = filemap_map_pages(vmf, start_pgoff, end_pgoff); xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED); + return ret; } static const struct vm_operations_struct xfs_file_vm_ops = { diff --git a/include/linux/mm.h b/include/linux/mm.h index ecdf8a8cd6ae..4572a9bc5862 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -542,8 +542,8 @@ struct vm_fault { * is not NULL, otherwise pmd. */ pgtable_t prealloc_pte; /* Pre-allocated pte page table. - * vm_ops->map_pages() calls - * alloc_set_pte() from atomic context. + * vm_ops->map_pages() sets up a page + * table from atomic context. * do_fault_around() pre-allocates * page table to avoid allocation from * atomic context. @@ -578,7 +578,7 @@ struct vm_operations_struct { vm_fault_t (*fault)(struct vm_fault *vmf); vm_fault_t (*huge_fault)(struct vm_fault *vmf, enum page_entry_size pe_size); - void (*map_pages)(struct vm_fault *vmf, + vm_fault_t (*map_pages)(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff); unsigned long (*pagesize)(struct vm_area_struct * area); @@ -988,7 +988,9 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) return pte; } -vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page); +vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page); +void do_set_pte(struct vm_fault *vmf, struct page *page); + vm_fault_t finish_fault(struct vm_fault *vmf); vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf); #endif @@ -2622,7 +2624,7 @@ extern void truncate_inode_pages_final(struct address_space *); /* generic vm_area_ops exported for stackable file systems */ extern vm_fault_t filemap_fault(struct vm_fault *vmf); -extern void filemap_map_pages(struct vm_fault *vmf, +extern vm_fault_t filemap_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff); extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf); diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 8fcdfa52eb4b..36eb748f3c97 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1314,6 +1314,17 @@ static inline int pmd_trans_unstable(pmd_t *pmd) #endif } +/* + * the ordering of these checks is important for pmds with _page_devmap set. + * if we check pmd_trans_unstable() first we will trip the bad_pmd() check + * inside of pmd_none_or_trans_huge_or_clear_bad(). this will end up correctly + * returning 1 but not before it spams dmesg with the pmd_clear_bad() output. + */ +static inline int pmd_devmap_trans_unstable(pmd_t *pmd) +{ + return pmd_devmap(*pmd) || pmd_trans_unstable(pmd); +} + #ifndef CONFIG_NUMA_BALANCING /* * Technically a PTE can be PROTNONE even when not doing NUMA balancing but diff --git a/mm/filemap.c b/mm/filemap.c index 5c9d564317a5..c1f2dc89b8a7 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -42,6 +42,7 @@ #include #include #include +#include #include "internal.h" #define CREATE_TRACE_POINTS @@ -2911,74 +2912,164 @@ out_retry: } EXPORT_SYMBOL(filemap_fault); -void filemap_map_pages(struct vm_fault *vmf, - pgoff_t start_pgoff, pgoff_t end_pgoff) +static bool filemap_map_pmd(struct vm_fault *vmf, struct page *page) { - struct file *file = vmf->vma->vm_file; + struct mm_struct *mm = vmf->vma->vm_mm; + + /* Huge page is mapped? No need to proceed. */ + if (pmd_trans_huge(*vmf->pmd)) { + unlock_page(page); + put_page(page); + return true; + } + + if (pmd_none(*vmf->pmd) && PageTransHuge(page)) { + vm_fault_t ret = do_set_pmd(vmf, page); + if (!ret) { + /* The page is mapped successfully, reference consumed. */ + unlock_page(page); + return true; + } + } + + if (pmd_none(*vmf->pmd)) { + vmf->ptl = pmd_lock(mm, vmf->pmd); + if (likely(pmd_none(*vmf->pmd))) { + mm_inc_nr_ptes(mm); + pmd_populate(mm, vmf->pmd, vmf->prealloc_pte); + vmf->prealloc_pte = NULL; + } + spin_unlock(vmf->ptl); + } + + /* See comment in handle_pte_fault() */ + if (pmd_devmap_trans_unstable(vmf->pmd)) { + unlock_page(page); + put_page(page); + return true; + } + + return false; +} + +static struct page *next_uptodate_page(struct page *page, + struct address_space *mapping, + struct xa_state *xas, pgoff_t end_pgoff) +{ + unsigned long max_idx; + + do { + if (!page) + return NULL; + if (xas_retry(xas, page)) + continue; + if (xa_is_value(page)) + continue; + if (PageLocked(page)) + continue; + if (!page_cache_get_speculative(page)) + continue; + /* Has the page moved or been split? */ + if (unlikely(page != xas_reload(xas))) + goto skip; + if (!PageUptodate(page) || PageReadahead(page)) + goto skip; + if (PageHWPoison(page)) + goto skip; + if (!trylock_page(page)) + goto skip; + if (page->mapping != mapping) + goto unlock; + if (!PageUptodate(page)) + goto unlock; + max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); + if (xas->xa_index >= max_idx) + goto unlock; + return page; +unlock: + unlock_page(page); +skip: + put_page(page); + } while ((page = xas_next_entry(xas, end_pgoff)) != NULL); + + return NULL; +} + +static inline struct page *first_map_page(struct address_space *mapping, + struct xa_state *xas, + pgoff_t end_pgoff) +{ + return next_uptodate_page(xas_find(xas, end_pgoff), + mapping, xas, end_pgoff); +} + +static inline struct page *next_map_page(struct address_space *mapping, + struct xa_state *xas, + pgoff_t end_pgoff) +{ + return next_uptodate_page(xas_next_entry(xas, end_pgoff), + mapping, xas, end_pgoff); +} + +vm_fault_t filemap_map_pages(struct vm_fault *vmf, + pgoff_t start_pgoff, pgoff_t end_pgoff) +{ + struct vm_area_struct *vma = vmf->vma; + struct file *file = vma->vm_file; struct address_space *mapping = file->f_mapping; pgoff_t last_pgoff = start_pgoff; - unsigned long max_idx; + unsigned long address = vmf->address; XA_STATE(xas, &mapping->i_pages, start_pgoff); struct page *head, *page; unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss); + vm_fault_t ret = 0; rcu_read_lock(); - xas_for_each(&xas, head, end_pgoff) { - if (xas_retry(&xas, head)) - continue; - if (xa_is_value(head)) - goto next; + head = first_map_page(mapping, &xas, end_pgoff); + if (!head) + goto out; - /* - * Check for a locked page first, as a speculative - * reference may adversely influence page migration. - */ - if (PageLocked(head)) - goto next; - if (!page_cache_get_speculative(head)) - goto next; + if (filemap_map_pmd(vmf, head)) { + ret = VM_FAULT_NOPAGE; + goto out; + } - /* Has the page moved or been split? */ - if (unlikely(head != xas_reload(&xas))) - goto skip; + vmf->address = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); + vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); + do { page = find_subpage(head, xas.xa_index); - - if (!PageUptodate(head) || - PageReadahead(page) || - PageHWPoison(page)) - goto skip; - if (!trylock_page(head)) - goto skip; - - if (head->mapping != mapping || !PageUptodate(head)) - goto unlock; - - max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); - if (xas.xa_index >= max_idx) + if (PageHWPoison(page)) goto unlock; if (mmap_miss > 0) mmap_miss--; vmf->address += (xas.xa_index - last_pgoff) << PAGE_SHIFT; - if (vmf->pte) - vmf->pte += xas.xa_index - last_pgoff; + vmf->pte += xas.xa_index - last_pgoff; last_pgoff = xas.xa_index; - if (alloc_set_pte(vmf, page)) + + if (!pte_none(*vmf->pte)) goto unlock; + + do_set_pte(vmf, page); + /* no need to invalidate: a not-present page won't be cached */ + update_mmu_cache(vma, vmf->address, vmf->pte); unlock_page(head); - goto next; + + /* The fault is handled */ + if (vmf->address == address) + ret = VM_FAULT_NOPAGE; + continue; unlock: unlock_page(head); -skip: put_page(head); -next: - /* Huge page is mapped? No need to proceed. */ - if (pmd_trans_huge(*vmf->pmd)) - break; - } + } while ((head = next_map_page(mapping, &xas, end_pgoff)) != NULL); + pte_unmap_unlock(vmf->pte, vmf->ptl); +out: rcu_read_unlock(); + vmf->address = address; WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss); + return ret; } EXPORT_SYMBOL(filemap_map_pages); diff --git a/mm/memory.c b/mm/memory.c index feff48e1465a..3e2fc2950ad7 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3503,7 +3503,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) if (pte_alloc(vma->vm_mm, vmf->pmd)) return VM_FAULT_OOM; - /* See the comment in pte_alloc_one_map() */ + /* See comment in handle_pte_fault() */ if (unlikely(pmd_trans_unstable(vmf->pmd))) return 0; @@ -3643,66 +3643,6 @@ static vm_fault_t __do_fault(struct vm_fault *vmf) return ret; } -/* - * The ordering of these checks is important for pmds with _PAGE_DEVMAP set. - * If we check pmd_trans_unstable() first we will trip the bad_pmd() check - * inside of pmd_none_or_trans_huge_or_clear_bad(). This will end up correctly - * returning 1 but not before it spams dmesg with the pmd_clear_bad() output. - */ -static int pmd_devmap_trans_unstable(pmd_t *pmd) -{ - return pmd_devmap(*pmd) || pmd_trans_unstable(pmd); -} - -static vm_fault_t pte_alloc_one_map(struct vm_fault *vmf) -{ - struct vm_area_struct *vma = vmf->vma; - - if (!pmd_none(*vmf->pmd)) - goto map_pte; - if (vmf->prealloc_pte) { - vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); - if (unlikely(!pmd_none(*vmf->pmd))) { - spin_unlock(vmf->ptl); - goto map_pte; - } - - mm_inc_nr_ptes(vma->vm_mm); - pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte); - spin_unlock(vmf->ptl); - vmf->prealloc_pte = NULL; - } else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd))) { - return VM_FAULT_OOM; - } -map_pte: - /* - * If a huge pmd materialized under us just retry later. Use - * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead of - * pmd_trans_huge() to ensure the pmd didn't become pmd_trans_huge - * under us and then back to pmd_none, as a result of MADV_DONTNEED - * running immediately after a huge pmd fault in a different thread of - * this mm, in turn leading to a misleading pmd_trans_huge() retval. - * All we have to ensure is that it is a regular pmd that we can walk - * with pte_offset_map() and we can do that through an atomic read in - * C, which is what pmd_trans_unstable() provides. - */ - if (pmd_devmap_trans_unstable(vmf->pmd)) - return VM_FAULT_NOPAGE; - - /* - * At this point we know that our vmf->pmd points to a page of ptes - * and it cannot become pmd_none(), pmd_devmap() or pmd_trans_huge() - * for the duration of the fault. If a racing MADV_DONTNEED runs and - * we zap the ptes pointed to by our vmf->pmd, the vmf->ptl will still - * be valid and we will re-check to make sure the vmf->pte isn't - * pte_none() under vmf->ptl protection when we return to - * alloc_set_pte(). - */ - vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, - &vmf->ptl); - return 0; -} - #ifdef CONFIG_TRANSPARENT_HUGEPAGE static void deposit_prealloc_pte(struct vm_fault *vmf) { @@ -3717,7 +3657,7 @@ static void deposit_prealloc_pte(struct vm_fault *vmf) vmf->prealloc_pte = NULL; } -static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) +vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) { struct vm_area_struct *vma = vmf->vma; bool write = vmf->flags & FAULT_FLAG_WRITE; @@ -3775,52 +3715,17 @@ out: return ret; } #else -static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) +vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) { - BUILD_BUG(); - return 0; + return VM_FAULT_FALLBACK; } #endif -/** - * alloc_set_pte - setup new PTE entry for given page and add reverse page - * mapping. If needed, the function allocates page table or use pre-allocated. - * - * @vmf: fault environment - * @page: page to map - * - * Caller must take care of unlocking vmf->ptl, if vmf->pte is non-NULL on - * return. - * - * Target users are page handler itself and implementations of - * vm_ops->map_pages. - * - * Return: %0 on success, %VM_FAULT_ code in case of error. - */ -vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page) +void do_set_pte(struct vm_fault *vmf, struct page *page) { struct vm_area_struct *vma = vmf->vma; bool write = vmf->flags & FAULT_FLAG_WRITE; pte_t entry; - vm_fault_t ret; - - if (pmd_none(*vmf->pmd) && PageTransCompound(page)) { - ret = do_set_pmd(vmf, page); - if (ret != VM_FAULT_FALLBACK) - return ret; - } - - if (!vmf->pte) { - ret = pte_alloc_one_map(vmf); - if (ret) - return ret; - } - - /* Re-check under ptl */ - if (unlikely(!pte_none(*vmf->pte))) { - update_mmu_tlb(vma, vmf->address, vmf->pte); - return VM_FAULT_NOPAGE; - } flush_icache_page(vma, page); entry = mk_pte(page, vma->vm_page_prot); @@ -3837,14 +3742,8 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page) page_add_file_rmap(page, false); } set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry); - - /* no need to invalidate: a not-present page won't be cached */ - update_mmu_cache(vma, vmf->address, vmf->pte); - - return 0; } - /** * finish_fault - finish page fault once we have prepared the page to fault * @@ -3862,12 +3761,12 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page) */ vm_fault_t finish_fault(struct vm_fault *vmf) { + struct vm_area_struct *vma = vmf->vma; struct page *page; - vm_fault_t ret = 0; + vm_fault_t ret; /* Did we COW the page? */ - if ((vmf->flags & FAULT_FLAG_WRITE) && - !(vmf->vma->vm_flags & VM_SHARED)) + if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) page = vmf->cow_page; else page = vmf->page; @@ -3876,12 +3775,38 @@ vm_fault_t finish_fault(struct vm_fault *vmf) * check even for read faults because we might have lost our CoWed * page */ - if (!(vmf->vma->vm_flags & VM_SHARED)) - ret = check_stable_address_space(vmf->vma->vm_mm); - if (!ret) - ret = alloc_set_pte(vmf, page); - if (vmf->pte) - pte_unmap_unlock(vmf->pte, vmf->ptl); + if (!(vma->vm_flags & VM_SHARED)) { + ret = check_stable_address_space(vma->vm_mm); + if (ret) + return ret; + } + + if (pmd_none(*vmf->pmd)) { + if (PageTransCompound(page)) { + ret = do_set_pmd(vmf, page); + if (ret != VM_FAULT_FALLBACK) + return ret; + } + + if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd))) + return VM_FAULT_OOM; + } + + /* See comment in handle_pte_fault() */ + if (pmd_devmap_trans_unstable(vmf->pmd)) + return 0; + + vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, + vmf->address, &vmf->ptl); + ret = 0; + /* Re-check under ptl */ + if (likely(pte_none(*vmf->pte))) + do_set_pte(vmf, page); + else + ret = VM_FAULT_NOPAGE; + + update_mmu_tlb(vma, vmf->address, vmf->pte); + pte_unmap_unlock(vmf->pte, vmf->ptl); return ret; } @@ -3951,13 +3876,12 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) pgoff_t start_pgoff = vmf->pgoff; pgoff_t end_pgoff; int off; - vm_fault_t ret = 0; nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT; mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK; - vmf->address = max(address & mask, vmf->vma->vm_start); - off = ((address - vmf->address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); + address = max(address & mask, vmf->vma->vm_start); + off = ((vmf->address - address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); start_pgoff -= off; /* @@ -3965,7 +3889,7 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) * the vma or nr_pages from start_pgoff, depending what is nearest. */ end_pgoff = start_pgoff - - ((vmf->address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) + + ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) + PTRS_PER_PTE - 1; end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1, start_pgoff + nr_pages - 1); @@ -3973,31 +3897,11 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) if (pmd_none(*vmf->pmd)) { vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm); if (!vmf->prealloc_pte) - goto out; + return VM_FAULT_OOM; smp_wmb(); /* See comment in __pte_alloc() */ } - vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff); - - /* Huge page is mapped? Page fault is solved */ - if (pmd_trans_huge(*vmf->pmd)) { - ret = VM_FAULT_NOPAGE; - goto out; - } - - /* ->map_pages() haven't done anything useful. Cold page cache? */ - if (!vmf->pte) - goto out; - - /* check if the page fault is solved */ - vmf->pte -= (vmf->address >> PAGE_SHIFT) - (address >> PAGE_SHIFT); - if (!pte_none(*vmf->pte)) - ret = VM_FAULT_NOPAGE; - pte_unmap_unlock(vmf->pte, vmf->ptl); -out: - vmf->address = address; - vmf->pte = NULL; - return ret; + return vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff); } static vm_fault_t do_read_fault(struct vm_fault *vmf) @@ -4353,7 +4257,18 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) */ vmf->pte = NULL; } else { - /* See comment in pte_alloc_one_map() */ + /* + * If a huge pmd materialized under us just retry later. Use + * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead + * of pmd_trans_huge() to ensure the pmd didn't become + * pmd_trans_huge under us and then back to pmd_none, as a + * result of MADV_DONTNEED running immediately after a huge pmd + * fault in a different thread of this mm, in turn leading to a + * misleading pmd_trans_huge() retval. All we have to ensure is + * that it is a regular pmd that we can walk with + * pte_offset_map() and we can do that through an atomic read + * in C, which is what pmd_trans_unstable() provides. + */ if (pmd_devmap_trans_unstable(vmf->pmd)) return 0; /* -- cgit v1.2.3 From 46bdb4277f98e70d0c91f4289897ade533fe9e80 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 24 Nov 2020 18:48:26 +0000 Subject: mm: Allow architectures to request 'old' entries when prefaulting Commit 5c0a85fad949 ("mm: make faultaround produce old ptes") changed the "faultaround" behaviour to initialise prefaulted PTEs as 'old', since this avoids vmscan wrongly assuming that they are hot, despite having never been explicitly accessed by userspace. The change has been shown to benefit numerous arm64 micro-architectures (with hardware access flag) running Android, where both application launch latency and direct reclaim time are significantly reduced (by 10%+ and ~80% respectively). Unfortunately, commit 315d09bf30c2 ("Revert "mm: make faultaround produce old ptes"") reverted the change due to it being identified as the cause of a ~6% regression in unixbench on x86. Experiments on a variety of recent arm64 micro-architectures indicate that unixbench is not affected by the original commit, which appears to yield a 0-1% performance improvement. Since one size does not fit all for the initial state of prefaulted PTEs, introduce arch_wants_old_prefaulted_pte(), which allows an architecture to opt-in to 'old' prefaulted PTEs at runtime based on whatever criteria it may have. Cc: Jan Kara Cc: Minchan Kim Cc: Andrew Morton Cc: Kirill A. Shutemov Cc: Linus Torvalds Reported-by: Vinayak Menon Signed-off-by: Will Deacon --- include/linux/mm.h | 5 ++++- mm/filemap.c | 14 ++++++++++---- mm/memory.c | 20 +++++++++++++++++++- 3 files changed, 33 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 4572a9bc5862..251a2339befb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -434,6 +434,7 @@ extern pgprot_t protection_map[16]; * @FAULT_FLAG_REMOTE: The fault is not for current task/mm. * @FAULT_FLAG_INSTRUCTION: The fault was during an instruction fetch. * @FAULT_FLAG_INTERRUPTIBLE: The fault can be interrupted by non-fatal signals. + * @FAULT_FLAG_PREFAULT: Fault was a prefault. * * About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify * whether we would allow page faults to retry by specifying these two @@ -464,6 +465,7 @@ extern pgprot_t protection_map[16]; #define FAULT_FLAG_REMOTE 0x80 #define FAULT_FLAG_INSTRUCTION 0x100 #define FAULT_FLAG_INTERRUPTIBLE 0x200 +#define FAULT_FLAG_PREFAULT 0x400 /* * The default fault flags that should be used by most of the @@ -501,7 +503,8 @@ static inline bool fault_flag_allow_retry_first(unsigned int flags) { FAULT_FLAG_USER, "USER" }, \ { FAULT_FLAG_REMOTE, "REMOTE" }, \ { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ - { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" } + { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ + { FAULT_FLAG_PREFAULT, "PREFAULT" } /* * vm_fault is filled by the pagefault handler and passed to the vma's diff --git a/mm/filemap.c b/mm/filemap.c index c1f2dc89b8a7..a6dc97906c8e 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3019,6 +3019,7 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, struct address_space *mapping = file->f_mapping; pgoff_t last_pgoff = start_pgoff; unsigned long address = vmf->address; + unsigned long flags = vmf->flags; XA_STATE(xas, &mapping->i_pages, start_pgoff); struct page *head, *page; unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss); @@ -3051,14 +3052,18 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, if (!pte_none(*vmf->pte)) goto unlock; + /* We're about to handle the fault */ + if (vmf->address == address) { + vmf->flags &= ~FAULT_FLAG_PREFAULT; + ret = VM_FAULT_NOPAGE; + } else { + vmf->flags |= FAULT_FLAG_PREFAULT; + } + do_set_pte(vmf, page); /* no need to invalidate: a not-present page won't be cached */ update_mmu_cache(vma, vmf->address, vmf->pte); unlock_page(head); - - /* The fault is handled */ - if (vmf->address == address) - ret = VM_FAULT_NOPAGE; continue; unlock: unlock_page(head); @@ -3067,6 +3072,7 @@ unlock: pte_unmap_unlock(vmf->pte, vmf->ptl); out: rcu_read_unlock(); + vmf->flags = flags; vmf->address = address; WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss); return ret; diff --git a/mm/memory.c b/mm/memory.c index 3e2fc2950ad7..f0e7c589ca9d 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -134,6 +134,18 @@ static inline bool arch_faults_on_old_pte(void) } #endif +#ifndef arch_wants_old_prefaulted_pte +static inline bool arch_wants_old_prefaulted_pte(void) +{ + /* + * Transitioning a PTE from 'old' to 'young' can be expensive on + * some architectures, even if it's performed in hardware. By + * default, "false" means prefaulted entries will be 'young'. + */ + return false; +} +#endif + static int __init disable_randmaps(char *s) { randomize_va_space = 0; @@ -3725,11 +3737,17 @@ void do_set_pte(struct vm_fault *vmf, struct page *page) { struct vm_area_struct *vma = vmf->vma; bool write = vmf->flags & FAULT_FLAG_WRITE; + bool prefault = vmf->flags & FAULT_FLAG_PREFAULT; pte_t entry; flush_icache_page(vma, page); entry = mk_pte(page, vma->vm_page_prot); - entry = pte_sw_mkyoung(entry); + + if (prefault && arch_wants_old_prefaulted_pte()) + entry = pte_mkold(entry); + else + entry = pte_sw_mkyoung(entry); + if (write) entry = maybe_mkwrite(pte_mkdirty(entry), vma); /* copy-on-write page */ -- cgit v1.2.3 From 149ae80b1d50e7db5ac7df1cdf0820017b70e716 Mon Sep 17 00:00:00 2001 From: Rafał Miłecki Date: Thu, 14 Jan 2021 11:53:18 +0100 Subject: soc: bcm: brcmstb: add stubs for getting platform IDs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some brcmstb drivers may be shared with other SoC families. E.g. the same USB PHY block is shared by brcmstb and BCM4908. To avoid building brcmstb common code on non-brcmstb platforms we need stubs for: 1. brcmstb_get_family_id() 2. brcmstb_get_product_id() (to avoid "undefined reference to" errors). With this change PHY_BRCM_USB will not have to unconditionally select SOC_BRCMSTB anymore. Signed-off-by: Rafał Miłecki Signed-off-by: Florian Fainelli --- include/linux/soc/brcmstb/brcmstb.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'include/linux') diff --git a/include/linux/soc/brcmstb/brcmstb.h b/include/linux/soc/brcmstb/brcmstb.h index 8e884e0dda0a..f2b768852777 100644 --- a/include/linux/soc/brcmstb/brcmstb.h +++ b/include/linux/soc/brcmstb/brcmstb.h @@ -2,6 +2,8 @@ #ifndef __BRCMSTB_SOC_H #define __BRCMSTB_SOC_H +#include + static inline u32 BRCM_ID(u32 reg) { return reg >> 28 ? reg >> 16 : reg >> 8; @@ -12,6 +14,8 @@ static inline u32 BRCM_REV(u32 reg) return reg & 0xff; } +#if IS_ENABLED(CONFIG_SOC_BRCMSTB) + /* * Helper functions for getting family or product id from the * SoC driver. @@ -19,4 +23,16 @@ static inline u32 BRCM_REV(u32 reg) u32 brcmstb_get_family_id(void); u32 brcmstb_get_product_id(void); +#else +static inline u32 brcmstb_get_family_id(void) +{ + return 0; +} + +static inline u32 brcmstb_get_product_id(void) +{ + return 0; +} +#endif + #endif /* __BRCMSTB_SOC_H */ -- cgit v1.2.3 From 9cacf81f8161111db25f98e78a7a0e32ae142b3f Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Fri, 15 Jan 2021 08:34:59 -0800 Subject: bpf: Remove extra lock_sock for TCP_ZEROCOPY_RECEIVE Add custom implementation of getsockopt hook for TCP_ZEROCOPY_RECEIVE. We skip generic hooks for TCP_ZEROCOPY_RECEIVE and have a custom call in do_tcp_getsockopt using the on-stack data. This removes 3% overhead for locking/unlocking the socket. Without this patch: 3.38% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt | --3.30%--__cgroup_bpf_run_filter_getsockopt | --0.81%--__kmalloc With the patch applied: 0.52% 0.12% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt_kern Note, exporting uapi/tcp.h requires removing netinet/tcp.h from test_progs.h because those headers have confliciting definitions. Signed-off-by: Stanislav Fomichev Signed-off-by: Alexei Starovoitov Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/bpf/20210115163501.805133-2-sdf@google.com --- include/linux/bpf-cgroup.h | 27 +- include/linux/indirect_call_wrapper.h | 6 + include/net/sock.h | 2 + include/net/tcp.h | 1 + kernel/bpf/cgroup.c | 46 +++ net/ipv4/tcp.c | 14 + net/ipv4/tcp_ipv4.c | 1 + net/ipv6/tcp_ipv6.c | 1 + net/socket.c | 3 + tools/include/uapi/linux/tcp.h | 357 +++++++++++++++++++++ .../testing/selftests/bpf/prog_tests/bpf_tcp_ca.c | 1 + .../selftests/bpf/prog_tests/cls_redirect.c | 1 + .../selftests/bpf/prog_tests/sockmap_basic.c | 1 + .../testing/selftests/bpf/prog_tests/sockopt_sk.c | 28 ++ tools/testing/selftests/bpf/progs/sockopt_sk.c | 23 +- tools/testing/selftests/bpf/test_progs.h | 1 - 16 files changed, 506 insertions(+), 7 deletions(-) create mode 100644 tools/include/uapi/linux/tcp.h (limited to 'include/linux') diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 72e69a0e1e8c..bcb2915e6124 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -147,6 +147,10 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, int __user *optlen, int max_optlen, int retval); +int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, + int optname, void *optval, + int *optlen, int retval); + static inline enum bpf_cgroup_storage_type cgroup_storage_type( struct bpf_map *map) { @@ -364,10 +368,23 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, ({ \ int __ret = retval; \ if (cgroup_bpf_enabled) \ - __ret = __cgroup_bpf_run_filter_getsockopt(sock, level, \ - optname, optval, \ - optlen, max_optlen, \ - retval); \ + if (!(sock)->sk_prot->bpf_bypass_getsockopt || \ + !INDIRECT_CALL_INET_1((sock)->sk_prot->bpf_bypass_getsockopt, \ + tcp_bpf_bypass_getsockopt, \ + level, optname)) \ + __ret = __cgroup_bpf_run_filter_getsockopt( \ + sock, level, optname, optval, optlen, \ + max_optlen, retval); \ + __ret; \ +}) + +#define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval, \ + optlen, retval) \ +({ \ + int __ret = retval; \ + if (cgroup_bpf_enabled) \ + __ret = __cgroup_bpf_run_filter_getsockopt_kern( \ + sock, level, optname, optval, optlen, retval); \ __ret; \ }) @@ -452,6 +469,8 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, #define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; }) #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \ optlen, max_optlen, retval) ({ retval; }) +#define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval, \ + optlen, retval) ({ retval; }) #define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \ kernel_optval) ({ 0; }) diff --git a/include/linux/indirect_call_wrapper.h b/include/linux/indirect_call_wrapper.h index 54c02c84906a..cfcfef37b2f1 100644 --- a/include/linux/indirect_call_wrapper.h +++ b/include/linux/indirect_call_wrapper.h @@ -60,4 +60,10 @@ #define INDIRECT_CALL_INET(f, f2, f1, ...) f(__VA_ARGS__) #endif +#if IS_ENABLED(CONFIG_INET) +#define INDIRECT_CALL_INET_1(f, f1, ...) INDIRECT_CALL_1(f, f1, __VA_ARGS__) +#else +#define INDIRECT_CALL_INET_1(f, f1, ...) f(__VA_ARGS__) +#endif + #endif diff --git a/include/net/sock.h b/include/net/sock.h index 129d200bccb4..7644ea64a376 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -1174,6 +1174,8 @@ struct proto { int (*backlog_rcv) (struct sock *sk, struct sk_buff *skb); + bool (*bpf_bypass_getsockopt)(int level, + int optname); void (*release_cb)(struct sock *sk); diff --git a/include/net/tcp.h b/include/net/tcp.h index 78d13c88720f..4bb42fb19711 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -403,6 +403,7 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, struct poll_table_struct *wait); int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen); +bool tcp_bpf_bypass_getsockopt(int level, int optname); int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, unsigned int optlen); void tcp_set_keepalive(struct sock *sk, int val); diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 96555a8a2c54..416e7738981b 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1486,6 +1486,52 @@ out: sockopt_free_buf(&ctx); return ret; } + +int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, + int optname, void *optval, + int *optlen, int retval) +{ + struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); + struct bpf_sockopt_kern ctx = { + .sk = sk, + .level = level, + .optname = optname, + .retval = retval, + .optlen = *optlen, + .optval = optval, + .optval_end = optval + *optlen, + }; + int ret; + + /* Note that __cgroup_bpf_run_filter_getsockopt doesn't copy + * user data back into BPF buffer when reval != 0. This is + * done as an optimization to avoid extra copy, assuming + * kernel won't populate the data in case of an error. + * Here we always pass the data and memset() should + * be called if that data shouldn't be "exported". + */ + + ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT], + &ctx, BPF_PROG_RUN); + if (!ret) + return -EPERM; + + if (ctx.optlen > *optlen) + return -EFAULT; + + /* BPF programs only allowed to set retval to 0, not some + * arbitrary value. + */ + if (ctx.retval != 0 && ctx.retval != retval) + return -EFAULT; + + /* BPF programs can shrink the buffer, export the modifications. + */ + if (ctx.optlen != 0) + *optlen = ctx.optlen; + + return ctx.retval; +} #endif static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp, diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 856ae516ac18..26aa923cf522 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -4099,6 +4099,8 @@ static int do_tcp_getsockopt(struct sock *sk, int level, return -EFAULT; lock_sock(sk); err = tcp_zerocopy_receive(sk, &zc); + err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, + &zc, &len, err); release_sock(sk); if (len >= offsetofend(struct tcp_zerocopy_receive, err)) goto zerocopy_rcv_sk_err; @@ -4133,6 +4135,18 @@ zerocopy_rcv_out: return 0; } +bool tcp_bpf_bypass_getsockopt(int level, int optname) +{ + /* TCP do_tcp_getsockopt has optimized getsockopt implementation + * to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE. + */ + if (level == SOL_TCP && optname == TCP_ZEROCOPY_RECEIVE) + return true; + + return false; +} +EXPORT_SYMBOL(tcp_bpf_bypass_getsockopt); + int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen) { diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 777306b5bc22..62b6fd385a47 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -2793,6 +2793,7 @@ struct proto tcp_prot = { .shutdown = tcp_shutdown, .setsockopt = tcp_setsockopt, .getsockopt = tcp_getsockopt, + .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt, .keepalive = tcp_set_keepalive, .recvmsg = tcp_recvmsg, .sendmsg = tcp_sendmsg, diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 0e1509b02cb3..8539715ff035 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -2121,6 +2121,7 @@ struct proto tcpv6_prot = { .shutdown = tcp_shutdown, .setsockopt = tcp_setsockopt, .getsockopt = tcp_getsockopt, + .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt, .keepalive = tcp_set_keepalive, .recvmsg = tcp_recvmsg, .sendmsg = tcp_sendmsg, diff --git a/net/socket.c b/net/socket.c index 33e8b6c4e1d3..7f0617ab5437 100644 --- a/net/socket.c +++ b/net/socket.c @@ -2126,6 +2126,9 @@ SYSCALL_DEFINE5(setsockopt, int, fd, int, level, int, optname, return __sys_setsockopt(fd, level, optname, optval, optlen); } +INDIRECT_CALLABLE_DECLARE(bool tcp_bpf_bypass_getsockopt(int level, + int optname)); + /* * Get a socket option. Because we don't know the option lengths we have * to pass a user mode parameter for the protocols to sort out. diff --git a/tools/include/uapi/linux/tcp.h b/tools/include/uapi/linux/tcp.h new file mode 100644 index 000000000000..13ceeb395eb8 --- /dev/null +++ b/tools/include/uapi/linux/tcp.h @@ -0,0 +1,357 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* + * INET An implementation of the TCP/IP protocol suite for the LINUX + * operating system. INET is implemented using the BSD Socket + * interface as the means of communication with the user level. + * + * Definitions for the TCP protocol. + * + * Version: @(#)tcp.h 1.0.2 04/28/93 + * + * Author: Fred N. van Kempen, + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ +#ifndef _UAPI_LINUX_TCP_H +#define _UAPI_LINUX_TCP_H + +#include +#include +#include + +struct tcphdr { + __be16 source; + __be16 dest; + __be32 seq; + __be32 ack_seq; +#if defined(__LITTLE_ENDIAN_BITFIELD) + __u16 res1:4, + doff:4, + fin:1, + syn:1, + rst:1, + psh:1, + ack:1, + urg:1, + ece:1, + cwr:1; +#elif defined(__BIG_ENDIAN_BITFIELD) + __u16 doff:4, + res1:4, + cwr:1, + ece:1, + urg:1, + ack:1, + psh:1, + rst:1, + syn:1, + fin:1; +#else +#error "Adjust your defines" +#endif + __be16 window; + __sum16 check; + __be16 urg_ptr; +}; + +/* + * The union cast uses a gcc extension to avoid aliasing problems + * (union is compatible to any of its members) + * This means this part of the code is -fstrict-aliasing safe now. + */ +union tcp_word_hdr { + struct tcphdr hdr; + __be32 words[5]; +}; + +#define tcp_flag_word(tp) ( ((union tcp_word_hdr *)(tp))->words [3]) + +enum { + TCP_FLAG_CWR = __constant_cpu_to_be32(0x00800000), + TCP_FLAG_ECE = __constant_cpu_to_be32(0x00400000), + TCP_FLAG_URG = __constant_cpu_to_be32(0x00200000), + TCP_FLAG_ACK = __constant_cpu_to_be32(0x00100000), + TCP_FLAG_PSH = __constant_cpu_to_be32(0x00080000), + TCP_FLAG_RST = __constant_cpu_to_be32(0x00040000), + TCP_FLAG_SYN = __constant_cpu_to_be32(0x00020000), + TCP_FLAG_FIN = __constant_cpu_to_be32(0x00010000), + TCP_RESERVED_BITS = __constant_cpu_to_be32(0x0F000000), + TCP_DATA_OFFSET = __constant_cpu_to_be32(0xF0000000) +}; + +/* + * TCP general constants + */ +#define TCP_MSS_DEFAULT 536U /* IPv4 (RFC1122, RFC2581) */ +#define TCP_MSS_DESIRED 1220U /* IPv6 (tunneled), EDNS0 (RFC3226) */ + +/* TCP socket options */ +#define TCP_NODELAY 1 /* Turn off Nagle's algorithm. */ +#define TCP_MAXSEG 2 /* Limit MSS */ +#define TCP_CORK 3 /* Never send partially complete segments */ +#define TCP_KEEPIDLE 4 /* Start keeplives after this period */ +#define TCP_KEEPINTVL 5 /* Interval between keepalives */ +#define TCP_KEEPCNT 6 /* Number of keepalives before death */ +#define TCP_SYNCNT 7 /* Number of SYN retransmits */ +#define TCP_LINGER2 8 /* Life time of orphaned FIN-WAIT-2 state */ +#define TCP_DEFER_ACCEPT 9 /* Wake up listener only when data arrive */ +#define TCP_WINDOW_CLAMP 10 /* Bound advertised window */ +#define TCP_INFO 11 /* Information about this connection. */ +#define TCP_QUICKACK 12 /* Block/reenable quick acks */ +#define TCP_CONGESTION 13 /* Congestion control algorithm */ +#define TCP_MD5SIG 14 /* TCP MD5 Signature (RFC2385) */ +#define TCP_THIN_LINEAR_TIMEOUTS 16 /* Use linear timeouts for thin streams*/ +#define TCP_THIN_DUPACK 17 /* Fast retrans. after 1 dupack */ +#define TCP_USER_TIMEOUT 18 /* How long for loss retry before timeout */ +#define TCP_REPAIR 19 /* TCP sock is under repair right now */ +#define TCP_REPAIR_QUEUE 20 +#define TCP_QUEUE_SEQ 21 +#define TCP_REPAIR_OPTIONS 22 +#define TCP_FASTOPEN 23 /* Enable FastOpen on listeners */ +#define TCP_TIMESTAMP 24 +#define TCP_NOTSENT_LOWAT 25 /* limit number of unsent bytes in write queue */ +#define TCP_CC_INFO 26 /* Get Congestion Control (optional) info */ +#define TCP_SAVE_SYN 27 /* Record SYN headers for new connections */ +#define TCP_SAVED_SYN 28 /* Get SYN headers recorded for connection */ +#define TCP_REPAIR_WINDOW 29 /* Get/set window parameters */ +#define TCP_FASTOPEN_CONNECT 30 /* Attempt FastOpen with connect */ +#define TCP_ULP 31 /* Attach a ULP to a TCP connection */ +#define TCP_MD5SIG_EXT 32 /* TCP MD5 Signature with extensions */ +#define TCP_FASTOPEN_KEY 33 /* Set the key for Fast Open (cookie) */ +#define TCP_FASTOPEN_NO_COOKIE 34 /* Enable TFO without a TFO cookie */ +#define TCP_ZEROCOPY_RECEIVE 35 +#define TCP_INQ 36 /* Notify bytes available to read as a cmsg on read */ + +#define TCP_CM_INQ TCP_INQ + +#define TCP_TX_DELAY 37 /* delay outgoing packets by XX usec */ + + +#define TCP_REPAIR_ON 1 +#define TCP_REPAIR_OFF 0 +#define TCP_REPAIR_OFF_NO_WP -1 /* Turn off without window probes */ + +struct tcp_repair_opt { + __u32 opt_code; + __u32 opt_val; +}; + +struct tcp_repair_window { + __u32 snd_wl1; + __u32 snd_wnd; + __u32 max_window; + + __u32 rcv_wnd; + __u32 rcv_wup; +}; + +enum { + TCP_NO_QUEUE, + TCP_RECV_QUEUE, + TCP_SEND_QUEUE, + TCP_QUEUES_NR, +}; + +/* why fastopen failed from client perspective */ +enum tcp_fastopen_client_fail { + TFO_STATUS_UNSPEC, /* catch-all */ + TFO_COOKIE_UNAVAILABLE, /* if not in TFO_CLIENT_NO_COOKIE mode */ + TFO_DATA_NOT_ACKED, /* SYN-ACK did not ack SYN data */ + TFO_SYN_RETRANSMITTED, /* SYN-ACK did not ack SYN data after timeout */ +}; + +/* for TCP_INFO socket option */ +#define TCPI_OPT_TIMESTAMPS 1 +#define TCPI_OPT_SACK 2 +#define TCPI_OPT_WSCALE 4 +#define TCPI_OPT_ECN 8 /* ECN was negociated at TCP session init */ +#define TCPI_OPT_ECN_SEEN 16 /* we received at least one packet with ECT */ +#define TCPI_OPT_SYN_DATA 32 /* SYN-ACK acked data in SYN sent or rcvd */ + +/* + * Sender's congestion state indicating normal or abnormal situations + * in the last round of packets sent. The state is driven by the ACK + * information and timer events. + */ +enum tcp_ca_state { + /* + * Nothing bad has been observed recently. + * No apparent reordering, packet loss, or ECN marks. + */ + TCP_CA_Open = 0, +#define TCPF_CA_Open (1< +#include #include #include "bpf_dctcp.skel.h" #include "bpf_cubic.skel.h" diff --git a/tools/testing/selftests/bpf/prog_tests/cls_redirect.c b/tools/testing/selftests/bpf/prog_tests/cls_redirect.c index 9781d85cb223..e075d03ab630 100644 --- a/tools/testing/selftests/bpf/prog_tests/cls_redirect.c +++ b/tools/testing/selftests/bpf/prog_tests/cls_redirect.c @@ -7,6 +7,7 @@ #include #include +#include #include diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c index 85f73261fab0..b8b48cac2ac3 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2020 Cloudflare #include +#include #include "test_progs.h" #include "test_skmsg_load_helpers.skel.h" diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c b/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c index b25c9c45c148..d5b44b135c00 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c @@ -2,6 +2,12 @@ #include #include "cgroup_helpers.h" +#include + +#ifndef SOL_TCP +#define SOL_TCP IPPROTO_TCP +#endif + #define SOL_CUSTOM 0xdeadbeef static int getsetsockopt(void) @@ -11,6 +17,7 @@ static int getsetsockopt(void) char u8[4]; __u32 u32; char cc[16]; /* TCP_CA_NAME_MAX */ + struct tcp_zerocopy_receive zc; } buf = {}; socklen_t optlen; char *big_buf = NULL; @@ -154,6 +161,27 @@ static int getsetsockopt(void) goto err; } + /* TCP_ZEROCOPY_RECEIVE triggers */ + memset(&buf, 0, sizeof(buf)); + optlen = sizeof(buf.zc); + err = getsockopt(fd, SOL_TCP, TCP_ZEROCOPY_RECEIVE, &buf, &optlen); + if (err) { + log_err("Unexpected getsockopt(TCP_ZEROCOPY_RECEIVE) err=%d errno=%d", + err, errno); + goto err; + } + + memset(&buf, 0, sizeof(buf)); + buf.zc.address = 12345; /* rejected by BPF */ + optlen = sizeof(buf.zc); + errno = 0; + err = getsockopt(fd, SOL_TCP, TCP_ZEROCOPY_RECEIVE, &buf, &optlen); + if (errno != EPERM) { + log_err("Unexpected getsockopt(TCP_ZEROCOPY_RECEIVE) err=%d errno=%d", + err, errno); + goto err; + } + free(big_buf); close(fd); return 0; diff --git a/tools/testing/selftests/bpf/progs/sockopt_sk.c b/tools/testing/selftests/bpf/progs/sockopt_sk.c index 712df7b49cb1..d3597f81e6e9 100644 --- a/tools/testing/selftests/bpf/progs/sockopt_sk.c +++ b/tools/testing/selftests/bpf/progs/sockopt_sk.c @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 #include -#include -#include +#include #include +#include #include char _license[] SEC("license") = "GPL"; @@ -12,6 +12,10 @@ __u32 _version SEC("version") = 1; #define PAGE_SIZE 4096 #endif +#ifndef SOL_TCP +#define SOL_TCP IPPROTO_TCP +#endif + #define SOL_CUSTOM 0xdeadbeef struct sockopt_sk { @@ -57,6 +61,21 @@ int _getsockopt(struct bpf_sockopt *ctx) return 1; } + if (ctx->level == SOL_TCP && ctx->optname == TCP_ZEROCOPY_RECEIVE) { + /* Verify that TCP_ZEROCOPY_RECEIVE triggers. + * It has a custom implementation for performance + * reasons. + */ + + if (optval + sizeof(struct tcp_zerocopy_receive) > optval_end) + return 0; /* EPERM, bounds check */ + + if (((struct tcp_zerocopy_receive *)optval)->address != 0) + return 0; /* EPERM, unexpected data */ + + return 1; + } + if (ctx->level == SOL_IP && ctx->optname == IP_FREEBIND) { if (optval + 1 > optval_end) return 0; /* EPERM, bounds check */ diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index e49e2fdde942..f7c2fd89d01a 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h @@ -16,7 +16,6 @@ typedef __u16 __sum16; #include #include #include -#include #include #include #include -- cgit v1.2.3 From 20f2505fb436cfa674cf1f46aaa624f44d3d1d03 Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Fri, 15 Jan 2021 08:35:00 -0800 Subject: bpf: Try to avoid kzalloc in cgroup/{s,g}etsockopt When we attach a bpf program to cgroup/getsockopt any other getsockopt() syscall starts incurring kzalloc/kfree cost. Let add a small buffer on the stack and use it for small (majority) {s,g}etsockopt values. The buffer is small enough to fit into the cache line and cover the majority of simple options (most of them are 4 byte ints). It seems natural to do the same for setsockopt, but it's a bit more involved when the BPF program modifies the data (where we have to kmalloc). The assumption is that for the majority of setsockopt calls (which are doing pure BPF options or apply policy) this will bring some benefit as well. Without this patch (we remove about 1% __kmalloc): 3.38% 0.07% tcp_mmap [kernel.kallsyms] [k] __cgroup_bpf_run_filter_getsockopt | --3.30%--__cgroup_bpf_run_filter_getsockopt | --0.81%--__kmalloc Signed-off-by: Stanislav Fomichev Signed-off-by: Alexei Starovoitov Acked-by: Martin KaFai Lau Link: https://lore.kernel.org/bpf/20210115163501.805133-3-sdf@google.com --- include/linux/filter.h | 5 +++++ kernel/bpf/cgroup.c | 52 +++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 50 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/filter.h b/include/linux/filter.h index 7fdce5407214..5b3137d7b690 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1298,6 +1298,11 @@ struct bpf_sysctl_kern { u64 tmp_reg; }; +#define BPF_SOCKOPT_KERN_BUF_SIZE 32 +struct bpf_sockopt_buf { + u8 data[BPF_SOCKOPT_KERN_BUF_SIZE]; +}; + struct bpf_sockopt_kern { struct sock *sk; u8 *optval; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 416e7738981b..ba8a1199d0ba 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1298,7 +1298,8 @@ static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp, return empty; } -static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) +static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen, + struct bpf_sockopt_buf *buf) { if (unlikely(max_optlen < 0)) return -EINVAL; @@ -1310,6 +1311,15 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) max_optlen = PAGE_SIZE; } + if (max_optlen <= sizeof(buf->data)) { + /* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE + * bytes avoid the cost of kzalloc. + */ + ctx->optval = buf->data; + ctx->optval_end = ctx->optval + max_optlen; + return max_optlen; + } + ctx->optval = kzalloc(max_optlen, GFP_USER); if (!ctx->optval) return -ENOMEM; @@ -1319,16 +1329,26 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) return max_optlen; } -static void sockopt_free_buf(struct bpf_sockopt_kern *ctx) +static void sockopt_free_buf(struct bpf_sockopt_kern *ctx, + struct bpf_sockopt_buf *buf) { + if (ctx->optval == buf->data) + return; kfree(ctx->optval); } +static bool sockopt_buf_allocated(struct bpf_sockopt_kern *ctx, + struct bpf_sockopt_buf *buf) +{ + return ctx->optval != buf->data; +} + int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, int *optname, char __user *optval, int *optlen, char **kernel_optval) { struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); + struct bpf_sockopt_buf buf = {}; struct bpf_sockopt_kern ctx = { .sk = sk, .level = *level, @@ -1350,7 +1370,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, */ max_optlen = max_t(int, 16, *optlen); - max_optlen = sockopt_alloc_buf(&ctx, max_optlen); + max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf); if (max_optlen < 0) return max_optlen; @@ -1390,14 +1410,31 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, */ if (ctx.optlen != 0) { *optlen = ctx.optlen; - *kernel_optval = ctx.optval; + /* We've used bpf_sockopt_kern->buf as an intermediary + * storage, but the BPF program indicates that we need + * to pass this data to the kernel setsockopt handler. + * No way to export on-stack buf, have to allocate a + * new buffer. + */ + if (!sockopt_buf_allocated(&ctx, &buf)) { + void *p = kmalloc(ctx.optlen, GFP_USER); + + if (!p) { + ret = -ENOMEM; + goto out; + } + memcpy(p, ctx.optval, ctx.optlen); + *kernel_optval = p; + } else { + *kernel_optval = ctx.optval; + } /* export and don't free sockopt buf */ return 0; } } out: - sockopt_free_buf(&ctx); + sockopt_free_buf(&ctx, &buf); return ret; } @@ -1407,6 +1444,7 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, int retval) { struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); + struct bpf_sockopt_buf buf = {}; struct bpf_sockopt_kern ctx = { .sk = sk, .level = level, @@ -1425,7 +1463,7 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, ctx.optlen = max_optlen; - max_optlen = sockopt_alloc_buf(&ctx, max_optlen); + max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf); if (max_optlen < 0) return max_optlen; @@ -1483,7 +1521,7 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, ret = ctx.retval; out: - sockopt_free_buf(&ctx); + sockopt_free_buf(&ctx, &buf); return ret; } -- cgit v1.2.3 From a9ed15dae0755a0368735e0556a462d8519bdb05 Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Fri, 15 Jan 2021 08:35:01 -0800 Subject: bpf: Split cgroup_bpf_enabled per attach type When we attach any cgroup hook, the rest (even if unused/unattached) start to contribute small overhead. In particular, the one we want to avoid is __cgroup_bpf_run_filter_skb which does two redirections to get to the cgroup and pushes/pulls skb. Let's split cgroup_bpf_enabled to be per-attach to make sure only used attach types trigger. I've dropped some existing high-level cgroup_bpf_enabled in some places because BPF_PROG_CGROUP_XXX_RUN macros usually have another cgroup_bpf_enabled check. I also had to copy-paste BPF_CGROUP_RUN_SA_PROG_LOCK for GETPEERNAME/GETSOCKNAME because type for cgroup_bpf_enabled[type] has to be constant and known at compile time. Signed-off-by: Stanislav Fomichev Signed-off-by: Alexei Starovoitov Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20210115163501.805133-4-sdf@google.com --- include/linux/bpf-cgroup.h | 38 ++++++++++++++++++++------------------ kernel/bpf/cgroup.c | 14 ++++++-------- net/ipv4/af_inet.c | 9 +++++---- net/ipv4/udp.c | 7 +++---- net/ipv6/af_inet6.c | 9 +++++---- net/ipv6/udp.c | 7 +++---- 6 files changed, 42 insertions(+), 42 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index bcb2915e6124..0748fd87969e 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -23,8 +23,8 @@ struct ctl_table_header; #ifdef CONFIG_CGROUP_BPF -extern struct static_key_false cgroup_bpf_enabled_key; -#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key) +extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE]; +#define cgroup_bpf_enabled(type) static_branch_unlikely(&cgroup_bpf_enabled_key[type]) DECLARE_PER_CPU(struct bpf_cgroup_storage*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]); @@ -189,7 +189,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_INET_INGRESS)) \ __ret = __cgroup_bpf_run_filter_skb(sk, skb, \ BPF_CGROUP_INET_INGRESS); \ \ @@ -199,7 +199,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled && sk && sk == skb->sk) { \ + if (cgroup_bpf_enabled(BPF_CGROUP_INET_EGRESS) && sk && sk == skb->sk) { \ typeof(sk) __sk = sk_to_full_sk(sk); \ if (sk_fullsock(__sk)) \ __ret = __cgroup_bpf_run_filter_skb(__sk, skb, \ @@ -211,7 +211,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, #define BPF_CGROUP_RUN_SK_PROG(sk, type) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) { \ + if (cgroup_bpf_enabled(type)) { \ __ret = __cgroup_bpf_run_filter_sk(sk, type); \ } \ __ret; \ @@ -232,7 +232,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(type)) \ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ NULL); \ __ret; \ @@ -241,7 +241,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) { \ + if (cgroup_bpf_enabled(type)) { \ lock_sock(sk); \ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ t_ctx); \ @@ -256,8 +256,10 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, #define BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK(sk, uaddr) \ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_BIND, NULL) -#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (cgroup_bpf_enabled && \ - sk->sk_prot->pre_connect) +#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) \ + ((cgroup_bpf_enabled(BPF_CGROUP_INET4_CONNECT) || \ + cgroup_bpf_enabled(BPF_CGROUP_INET6_CONNECT)) && \ + (sk)->sk_prot->pre_connect) #define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) \ BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT) @@ -301,7 +303,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, #define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS)) \ __ret = __cgroup_bpf_run_filter_sock_ops(sk, \ sock_ops, \ BPF_CGROUP_SOCK_OPS); \ @@ -311,7 +313,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, #define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled && (sock_ops)->sk) { \ + if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS) && (sock_ops)->sk) { \ typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ if (__sk && sk_fullsock(__sk)) \ __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ @@ -324,7 +326,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_DEVICE)) \ __ret = __cgroup_bpf_check_dev_permission(type, major, minor, \ access, \ BPF_CGROUP_DEVICE); \ @@ -336,7 +338,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, #define BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, buf, count, pos) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_SYSCTL)) \ __ret = __cgroup_bpf_run_filter_sysctl(head, table, write, \ buf, count, pos, \ BPF_CGROUP_SYSCTL); \ @@ -347,7 +349,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, kernel_optval) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_SETSOCKOPT)) \ __ret = __cgroup_bpf_run_filter_setsockopt(sock, level, \ optname, optval, \ optlen, \ @@ -358,7 +360,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, #define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) \ ({ \ int __ret = 0; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \ get_user(__ret, optlen); \ __ret; \ }) @@ -367,7 +369,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, max_optlen, retval) \ ({ \ int __ret = retval; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \ if (!(sock)->sk_prot->bpf_bypass_getsockopt || \ !INDIRECT_CALL_INET_1((sock)->sk_prot->bpf_bypass_getsockopt, \ tcp_bpf_bypass_getsockopt, \ @@ -382,7 +384,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, optlen, retval) \ ({ \ int __ret = retval; \ - if (cgroup_bpf_enabled) \ + if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \ __ret = __cgroup_bpf_run_filter_getsockopt_kern( \ sock, level, optname, optval, optlen, retval); \ __ret; \ @@ -444,7 +446,7 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, return 0; } -#define cgroup_bpf_enabled (0) +#define cgroup_bpf_enabled(type) (0) #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) ({ 0; }) #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0) #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; }) diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index ba8a1199d0ba..da649f20d6b2 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -19,7 +19,7 @@ #include "../cgroup/cgroup-internal.h" -DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key); +DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_BPF_ATTACH_TYPE); EXPORT_SYMBOL(cgroup_bpf_enabled_key); void cgroup_bpf_offline(struct cgroup *cgrp) @@ -128,7 +128,7 @@ static void cgroup_bpf_release(struct work_struct *work) if (pl->link) bpf_cgroup_link_auto_detach(pl->link); kfree(pl); - static_branch_dec(&cgroup_bpf_enabled_key); + static_branch_dec(&cgroup_bpf_enabled_key[type]); } old_array = rcu_dereference_protected( cgrp->bpf.effective[type], @@ -499,7 +499,7 @@ int __cgroup_bpf_attach(struct cgroup *cgrp, if (old_prog) bpf_prog_put(old_prog); else - static_branch_inc(&cgroup_bpf_enabled_key); + static_branch_inc(&cgroup_bpf_enabled_key[type]); bpf_cgroup_storages_link(new_storage, cgrp, type); return 0; @@ -698,7 +698,7 @@ int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, cgrp->bpf.flags[type] = 0; if (old_prog) bpf_prog_put(old_prog); - static_branch_dec(&cgroup_bpf_enabled_key); + static_branch_dec(&cgroup_bpf_enabled_key[type]); return 0; cleanup: @@ -1360,8 +1360,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, * attached to the hook so we don't waste time allocating * memory and locking the socket. */ - if (!cgroup_bpf_enabled || - __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT)) + if (__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT)) return 0; /* Allocate a bit more than the initial user buffer for @@ -1457,8 +1456,7 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, * attached to the hook so we don't waste time allocating * memory and locking the socket. */ - if (!cgroup_bpf_enabled || - __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT)) + if (__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT)) return retval; ctx.optlen = max_optlen; diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index b94fa8eb831b..6ba2930ff49b 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -777,18 +777,19 @@ int inet_getname(struct socket *sock, struct sockaddr *uaddr, return -ENOTCONN; sin->sin_port = inet->inet_dport; sin->sin_addr.s_addr = inet->inet_daddr; + BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin, + BPF_CGROUP_INET4_GETPEERNAME, + NULL); } else { __be32 addr = inet->inet_rcv_saddr; if (!addr) addr = inet->inet_saddr; sin->sin_port = inet->inet_sport; sin->sin_addr.s_addr = addr; - } - if (cgroup_bpf_enabled) BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin, - peer ? BPF_CGROUP_INET4_GETPEERNAME : - BPF_CGROUP_INET4_GETSOCKNAME, + BPF_CGROUP_INET4_GETSOCKNAME, NULL); + } memset(sin->sin_zero, 0, sizeof(sin->sin_zero)); return sizeof(*sin); } diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 69ea76578abb..c67e483fce41 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -1124,7 +1124,7 @@ int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) rcu_read_unlock(); } - if (cgroup_bpf_enabled && !connected) { + if (cgroup_bpf_enabled(BPF_CGROUP_UDP4_SENDMSG) && !connected) { err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, (struct sockaddr *)usin, &ipc.addr); if (err) @@ -1858,9 +1858,8 @@ try_again: memset(sin->sin_zero, 0, sizeof(sin->sin_zero)); *addr_len = sizeof(*sin); - if (cgroup_bpf_enabled) - BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, - (struct sockaddr *)sin); + BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, + (struct sockaddr *)sin); } if (udp_sk(sk)->gro_enabled) diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 8e9c3e9ea36e..b9c654836b72 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -527,18 +527,19 @@ int inet6_getname(struct socket *sock, struct sockaddr *uaddr, sin->sin6_addr = sk->sk_v6_daddr; if (np->sndflow) sin->sin6_flowinfo = np->flow_label; + BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin, + BPF_CGROUP_INET6_GETPEERNAME, + NULL); } else { if (ipv6_addr_any(&sk->sk_v6_rcv_saddr)) sin->sin6_addr = np->saddr; else sin->sin6_addr = sk->sk_v6_rcv_saddr; sin->sin6_port = inet->inet_sport; - } - if (cgroup_bpf_enabled) BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin, - peer ? BPF_CGROUP_INET6_GETPEERNAME : - BPF_CGROUP_INET6_GETSOCKNAME, + BPF_CGROUP_INET6_GETSOCKNAME, NULL); + } sin->sin6_scope_id = ipv6_iface_scope_id(&sin->sin6_addr, sk->sk_bound_dev_if); return sizeof(*sin); diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index b9f3dfdd2383..a02ac875a923 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -409,9 +409,8 @@ try_again: } *addr_len = sizeof(*sin6); - if (cgroup_bpf_enabled) - BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, - (struct sockaddr *)sin6); + BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, + (struct sockaddr *)sin6); } if (udp_sk(sk)->gro_enabled) @@ -1462,7 +1461,7 @@ do_udp_sendmsg: fl6.saddr = np->saddr; fl6.fl6_sport = inet->inet_sport; - if (cgroup_bpf_enabled && !connected) { + if (cgroup_bpf_enabled(BPF_CGROUP_UDP6_SENDMSG) && !connected) { err = BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, (struct sockaddr *)sin6, &fl6.saddr); if (err) -- cgit v1.2.3 From 3b830a9c34d5897be07176ce4e6f2d75e2c8cfd7 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Mon, 18 Jan 2021 13:31:30 -0800 Subject: tty: convert tty_ldisc_ops 'read()' function to take a kernel pointer The tty line discipline .read() function was passed the final user pointer destination as an argument, which doesn't match the 'write()' function, and makes it very inconvenient to do a splice method for ttys. This is a conversion to use a kernel buffer instead. NOTE! It does this by passing the tty line discipline ->read() function an additional "cookie" to fill in, and an offset into the cookie data. The line discipline can fill in the cookie data with its own private information, and then the reader will repeat the read until either the cookie is cleared or it runs out of data. The only real user of this is N_HDLC, which can use this to handle big packets, even if the kernel buffer is smaller than the whole packet. Cc: Christoph Hellwig Cc: Greg Kroah-Hartman Cc: Al Viro Signed-off-by: Linus Torvalds --- drivers/bluetooth/hci_ldisc.c | 34 +++++++++--------- drivers/input/serio/serport.c | 4 ++- drivers/net/ppp/ppp_async.c | 3 +- drivers/net/ppp/ppp_synctty.c | 3 +- drivers/tty/n_gsm.c | 3 +- drivers/tty/n_hdlc.c | 60 +++++++++++++++++++++---------- drivers/tty/n_null.c | 3 +- drivers/tty/n_r3964.c | 10 +++--- drivers/tty/n_tracerouter.c | 4 ++- drivers/tty/n_tracesink.c | 4 ++- drivers/tty/n_tty.c | 82 ++++++++++++++++++------------------------- drivers/tty/tty_io.c | 64 +++++++++++++++++++++++++++++++-- include/linux/tty_ldisc.h | 3 +- net/nfc/nci/uart.c | 3 +- 14 files changed, 178 insertions(+), 102 deletions(-) (limited to 'include/linux') diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c index f83d67eafc9f..dd92aea15b8b 100644 --- a/drivers/bluetooth/hci_ldisc.c +++ b/drivers/bluetooth/hci_ldisc.c @@ -802,7 +802,8 @@ static int hci_uart_tty_ioctl(struct tty_struct *tty, struct file *file, * We don't provide read/write/poll interface for user space. */ static ssize_t hci_uart_tty_read(struct tty_struct *tty, struct file *file, - unsigned char __user *buf, size_t nr) + unsigned char *buf, size_t nr, + void **cookie, unsigned long offset) { return 0; } @@ -819,29 +820,28 @@ static __poll_t hci_uart_tty_poll(struct tty_struct *tty, return 0; } +static struct tty_ldisc_ops hci_uart_ldisc = { + .owner = THIS_MODULE, + .magic = TTY_LDISC_MAGIC, + .name = "n_hci", + .open = hci_uart_tty_open, + .close = hci_uart_tty_close, + .read = hci_uart_tty_read, + .write = hci_uart_tty_write, + .ioctl = hci_uart_tty_ioctl, + .compat_ioctl = hci_uart_tty_ioctl, + .poll = hci_uart_tty_poll, + .receive_buf = hci_uart_tty_receive, + .write_wakeup = hci_uart_tty_wakeup, +}; + static int __init hci_uart_init(void) { - static struct tty_ldisc_ops hci_uart_ldisc; int err; BT_INFO("HCI UART driver ver %s", VERSION); /* Register the tty discipline */ - - memset(&hci_uart_ldisc, 0, sizeof(hci_uart_ldisc)); - hci_uart_ldisc.magic = TTY_LDISC_MAGIC; - hci_uart_ldisc.name = "n_hci"; - hci_uart_ldisc.open = hci_uart_tty_open; - hci_uart_ldisc.close = hci_uart_tty_close; - hci_uart_ldisc.read = hci_uart_tty_read; - hci_uart_ldisc.write = hci_uart_tty_write; - hci_uart_ldisc.ioctl = hci_uart_tty_ioctl; - hci_uart_ldisc.compat_ioctl = hci_uart_tty_ioctl; - hci_uart_ldisc.poll = hci_uart_tty_poll; - hci_uart_ldisc.receive_buf = hci_uart_tty_receive; - hci_uart_ldisc.write_wakeup = hci_uart_tty_wakeup; - hci_uart_ldisc.owner = THIS_MODULE; - err = tty_register_ldisc(N_HCI, &hci_uart_ldisc); if (err) { BT_ERR("HCI line discipline registration failed. (%d)", err); diff --git a/drivers/input/serio/serport.c b/drivers/input/serio/serport.c index 8ac970a423de..33e9d9bfd036 100644 --- a/drivers/input/serio/serport.c +++ b/drivers/input/serio/serport.c @@ -156,7 +156,9 @@ out: * returning 0 characters. */ -static ssize_t serport_ldisc_read(struct tty_struct * tty, struct file * file, unsigned char __user * buf, size_t nr) +static ssize_t serport_ldisc_read(struct tty_struct * tty, struct file * file, + unsigned char *kbuf, size_t nr, + void **cookie, unsigned long offset) { struct serport *serport = (struct serport*) tty->disc_data; struct serio *serio; diff --git a/drivers/net/ppp/ppp_async.c b/drivers/net/ppp/ppp_async.c index 29a0917a81e6..f14a9d190de9 100644 --- a/drivers/net/ppp/ppp_async.c +++ b/drivers/net/ppp/ppp_async.c @@ -259,7 +259,8 @@ static int ppp_asynctty_hangup(struct tty_struct *tty) */ static ssize_t ppp_asynctty_read(struct tty_struct *tty, struct file *file, - unsigned char __user *buf, size_t count) + unsigned char *buf, size_t count, + void **cookie, unsigned long offset) { return -EAGAIN; } diff --git a/drivers/net/ppp/ppp_synctty.c b/drivers/net/ppp/ppp_synctty.c index 0f338752c38b..f774b7e52da4 100644 --- a/drivers/net/ppp/ppp_synctty.c +++ b/drivers/net/ppp/ppp_synctty.c @@ -257,7 +257,8 @@ static int ppp_sync_hangup(struct tty_struct *tty) */ static ssize_t ppp_sync_read(struct tty_struct *tty, struct file *file, - unsigned char __user *buf, size_t count) + unsigned char *buf, size_t count, + void **cookie, unsigned long offset) { return -EAGAIN; } diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c index 25f3152089c2..fea1eeac5b90 100644 --- a/drivers/tty/n_gsm.c +++ b/drivers/tty/n_gsm.c @@ -2557,7 +2557,8 @@ static void gsmld_write_wakeup(struct tty_struct *tty) */ static ssize_t gsmld_read(struct tty_struct *tty, struct file *file, - unsigned char __user *buf, size_t nr) + unsigned char *buf, size_t nr, + void **cookie, unsigned long offset) { return -EOPNOTSUPP; } diff --git a/drivers/tty/n_hdlc.c b/drivers/tty/n_hdlc.c index 12557ee1edb6..1363e659dc1d 100644 --- a/drivers/tty/n_hdlc.c +++ b/drivers/tty/n_hdlc.c @@ -416,13 +416,19 @@ static void n_hdlc_tty_receive(struct tty_struct *tty, const __u8 *data, * Returns the number of bytes returned or error code. */ static ssize_t n_hdlc_tty_read(struct tty_struct *tty, struct file *file, - __u8 __user *buf, size_t nr) + __u8 *kbuf, size_t nr, + void **cookie, unsigned long offset) { struct n_hdlc *n_hdlc = tty->disc_data; int ret = 0; struct n_hdlc_buf *rbuf; DECLARE_WAITQUEUE(wait, current); + /* Is this a repeated call for an rbuf we already found earlier? */ + rbuf = *cookie; + if (rbuf) + goto have_rbuf; + add_wait_queue(&tty->read_wait, &wait); for (;;) { @@ -436,25 +442,8 @@ static ssize_t n_hdlc_tty_read(struct tty_struct *tty, struct file *file, set_current_state(TASK_INTERRUPTIBLE); rbuf = n_hdlc_buf_get(&n_hdlc->rx_buf_list); - if (rbuf) { - if (rbuf->count > nr) { - /* too large for caller's buffer */ - ret = -EOVERFLOW; - } else { - __set_current_state(TASK_RUNNING); - if (copy_to_user(buf, rbuf->buf, rbuf->count)) - ret = -EFAULT; - else - ret = rbuf->count; - } - - if (n_hdlc->rx_free_buf_list.count > - DEFAULT_RX_BUF_COUNT) - kfree(rbuf); - else - n_hdlc_buf_put(&n_hdlc->rx_free_buf_list, rbuf); + if (rbuf) break; - } /* no data */ if (tty_io_nonblock(tty, file)) { @@ -473,6 +462,39 @@ static ssize_t n_hdlc_tty_read(struct tty_struct *tty, struct file *file, remove_wait_queue(&tty->read_wait, &wait); __set_current_state(TASK_RUNNING); + if (!rbuf) + return ret; + *cookie = rbuf; + +have_rbuf: + /* Have we used it up entirely? */ + if (offset >= rbuf->count) + goto done_with_rbuf; + + /* More data to go, but can't copy any more? EOVERFLOW */ + ret = -EOVERFLOW; + if (!nr) + goto done_with_rbuf; + + /* Copy as much data as possible */ + ret = rbuf->count - offset; + if (ret > nr) + ret = nr; + memcpy(kbuf, rbuf->buf+offset, ret); + offset += ret; + + /* If we still have data left, we leave the rbuf in the cookie */ + if (offset < rbuf->count) + return ret; + +done_with_rbuf: + *cookie = NULL; + + if (n_hdlc->rx_free_buf_list.count > DEFAULT_RX_BUF_COUNT) + kfree(rbuf); + else + n_hdlc_buf_put(&n_hdlc->rx_free_buf_list, rbuf); + return ret; } /* end of n_hdlc_tty_read() */ diff --git a/drivers/tty/n_null.c b/drivers/tty/n_null.c index 96feabae4740..ce03ae78f5c6 100644 --- a/drivers/tty/n_null.c +++ b/drivers/tty/n_null.c @@ -20,7 +20,8 @@ static void n_null_close(struct tty_struct *tty) } static ssize_t n_null_read(struct tty_struct *tty, struct file *file, - unsigned char __user * buf, size_t nr) + unsigned char *buf, size_t nr, + void **cookie, unsigned long offset) { return -EOPNOTSUPP; } diff --git a/drivers/tty/n_r3964.c b/drivers/tty/n_r3964.c index 934dd2fb2ec8..3161f0a535e3 100644 --- a/drivers/tty/n_r3964.c +++ b/drivers/tty/n_r3964.c @@ -129,7 +129,7 @@ static void remove_client_block(struct r3964_info *pInfo, static int r3964_open(struct tty_struct *tty); static void r3964_close(struct tty_struct *tty); static ssize_t r3964_read(struct tty_struct *tty, struct file *file, - unsigned char __user * buf, size_t nr); + void *cookie, unsigned char *buf, size_t nr); static ssize_t r3964_write(struct tty_struct *tty, struct file *file, const unsigned char *buf, size_t nr); static int r3964_ioctl(struct tty_struct *tty, struct file *file, @@ -1058,7 +1058,8 @@ static void r3964_close(struct tty_struct *tty) } static ssize_t r3964_read(struct tty_struct *tty, struct file *file, - unsigned char __user * buf, size_t nr) + unsigned char *kbuf, size_t nr, + void **cookie, unsigned long offset) { struct r3964_info *pInfo = tty->disc_data; struct r3964_client_info *pClient; @@ -1109,10 +1110,7 @@ static ssize_t r3964_read(struct tty_struct *tty, struct file *file, kfree(pMsg); TRACE_M("r3964_read - msg kfree %p", pMsg); - if (copy_to_user(buf, &theMsg, ret)) { - ret = -EFAULT; - goto unlock; - } + memcpy(kbuf, &theMsg, ret); TRACE_PS("read - return %d", ret); goto unlock; diff --git a/drivers/tty/n_tracerouter.c b/drivers/tty/n_tracerouter.c index 4479af4d2fa5..3490ed51b1a3 100644 --- a/drivers/tty/n_tracerouter.c +++ b/drivers/tty/n_tracerouter.c @@ -118,7 +118,9 @@ static void n_tracerouter_close(struct tty_struct *tty) * -EINVAL */ static ssize_t n_tracerouter_read(struct tty_struct *tty, struct file *file, - unsigned char __user *buf, size_t nr) { + unsigned char *buf, size_t nr, + void **cookie, unsigned long offset) +{ return -EINVAL; } diff --git a/drivers/tty/n_tracesink.c b/drivers/tty/n_tracesink.c index d96ba82cc356..1d9931041fd8 100644 --- a/drivers/tty/n_tracesink.c +++ b/drivers/tty/n_tracesink.c @@ -115,7 +115,9 @@ static void n_tracesink_close(struct tty_struct *tty) * -EINVAL */ static ssize_t n_tracesink_read(struct tty_struct *tty, struct file *file, - unsigned char __user *buf, size_t nr) { + unsigned char *buf, size_t nr, + void **cookie, unsigned long offset) +{ return -EINVAL; } diff --git a/drivers/tty/n_tty.c b/drivers/tty/n_tty.c index 7e5e36315260..4a34a9f43b29 100644 --- a/drivers/tty/n_tty.c +++ b/drivers/tty/n_tty.c @@ -164,29 +164,24 @@ static void zero_buffer(struct tty_struct *tty, u8 *buffer, int size) memset(buffer, 0x00, size); } -static int tty_copy_to_user(struct tty_struct *tty, void __user *to, - size_t tail, size_t n) +static void tty_copy(struct tty_struct *tty, void *to, size_t tail, size_t n) { struct n_tty_data *ldata = tty->disc_data; size_t size = N_TTY_BUF_SIZE - tail; void *from = read_buf_addr(ldata, tail); - int uncopied; if (n > size) { tty_audit_add_data(tty, from, size); - uncopied = copy_to_user(to, from, size); - zero_buffer(tty, from, size - uncopied); - if (uncopied) - return uncopied; + memcpy(to, from, size); + zero_buffer(tty, from, size); to += size; n -= size; from = ldata->read_buf; } tty_audit_add_data(tty, from, n); - uncopied = copy_to_user(to, from, n); - zero_buffer(tty, from, n - uncopied); - return uncopied; + memcpy(to, from, n); + zero_buffer(tty, from, n); } /** @@ -1942,15 +1937,16 @@ static inline int input_available_p(struct tty_struct *tty, int poll) /** * copy_from_read_buf - copy read data directly * @tty: terminal device - * @b: user data + * @kbp: data * @nr: size of data * * Helper function to speed up n_tty_read. It is only called when - * ICANON is off; it copies characters straight from the tty queue to - * user space directly. It can be profitably called twice; once to - * drain the space from the tail pointer to the (physical) end of the - * buffer, and once to drain the space from the (physical) beginning of - * the buffer to head pointer. + * ICANON is off; it copies characters straight from the tty queue. + * + * It can be profitably called twice; once to drain the space from + * the tail pointer to the (physical) end of the buffer, and once + * to drain the space from the (physical) beginning of the buffer + * to head pointer. * * Called under the ldata->atomic_read_lock sem * @@ -1960,7 +1956,7 @@ static inline int input_available_p(struct tty_struct *tty, int poll) */ static int copy_from_read_buf(struct tty_struct *tty, - unsigned char __user **b, + unsigned char **kbp, size_t *nr) { @@ -1976,8 +1972,7 @@ static int copy_from_read_buf(struct tty_struct *tty, n = min(*nr, n); if (n) { unsigned char *from = read_buf_addr(ldata, tail); - retval = copy_to_user(*b, from, n); - n -= retval; + memcpy(*kbp, from, n); is_eof = n == 1 && *from == EOF_CHAR(tty); tty_audit_add_data(tty, from, n); zero_buffer(tty, from, n); @@ -1986,7 +1981,7 @@ static int copy_from_read_buf(struct tty_struct *tty, if (L_EXTPROC(tty) && ldata->icanon && is_eof && (head == ldata->read_tail)) n = 0; - *b += n; + *kbp += n; *nr -= n; } return retval; @@ -1995,12 +1990,12 @@ static int copy_from_read_buf(struct tty_struct *tty, /** * canon_copy_from_read_buf - copy read data in canonical mode * @tty: terminal device - * @b: user data + * @kbp: data * @nr: size of data * * Helper function for n_tty_read. It is only called when ICANON is on; * it copies one line of input up to and including the line-delimiting - * character into the user-space buffer. + * character into the result buffer. * * NB: When termios is changed from non-canonical to canonical mode and * the read buffer contains data, n_tty_set_termios() simulates an EOF @@ -2016,14 +2011,14 @@ static int copy_from_read_buf(struct tty_struct *tty, */ static int canon_copy_from_read_buf(struct tty_struct *tty, - unsigned char __user **b, + unsigned char **kbp, size_t *nr) { struct n_tty_data *ldata = tty->disc_data; size_t n, size, more, c; size_t eol; size_t tail; - int ret, found = 0; + int found = 0; /* N.B. avoid overrun if nr == 0 */ if (!*nr) @@ -2059,10 +2054,8 @@ static int canon_copy_from_read_buf(struct tty_struct *tty, n_tty_trace("%s: eol:%zu found:%d n:%zu c:%zu tail:%zu more:%zu\n", __func__, eol, found, n, c, tail, more); - ret = tty_copy_to_user(tty, *b, tail, n); - if (ret) - return -EFAULT; - *b += n; + tty_copy(tty, *kbp, tail, n); + *kbp += n; *nr -= n; if (found) @@ -2130,10 +2123,11 @@ static int job_control(struct tty_struct *tty, struct file *file) */ static ssize_t n_tty_read(struct tty_struct *tty, struct file *file, - unsigned char __user *buf, size_t nr) + unsigned char *kbuf, size_t nr, + void **cookie, unsigned long offset) { struct n_tty_data *ldata = tty->disc_data; - unsigned char __user *b = buf; + unsigned char *kb = kbuf; DEFINE_WAIT_FUNC(wait, woken_wake_function); int c; int minimum, time; @@ -2179,17 +2173,13 @@ static ssize_t n_tty_read(struct tty_struct *tty, struct file *file, /* First test for status change. */ if (packet && tty->link->ctrl_status) { unsigned char cs; - if (b != buf) + if (kb != kbuf) break; spin_lock_irq(&tty->link->ctrl_lock); cs = tty->link->ctrl_status; tty->link->ctrl_status = 0; spin_unlock_irq(&tty->link->ctrl_lock); - if (put_user(cs, b)) { - retval = -EFAULT; - break; - } - b++; + *kb++ = cs; nr--; break; } @@ -2232,24 +2222,20 @@ static ssize_t n_tty_read(struct tty_struct *tty, struct file *file, } if (ldata->icanon && !L_EXTPROC(tty)) { - retval = canon_copy_from_read_buf(tty, &b, &nr); + retval = canon_copy_from_read_buf(tty, &kb, &nr); if (retval) break; } else { int uncopied; /* Deal with packet mode. */ - if (packet && b == buf) { - if (put_user(TIOCPKT_DATA, b)) { - retval = -EFAULT; - break; - } - b++; + if (packet && kb == kbuf) { + *kb++ = TIOCPKT_DATA; nr--; } - uncopied = copy_from_read_buf(tty, &b, &nr); - uncopied += copy_from_read_buf(tty, &b, &nr); + uncopied = copy_from_read_buf(tty, &kb, &nr); + uncopied += copy_from_read_buf(tty, &kb, &nr); if (uncopied) { retval = -EFAULT; break; @@ -2258,7 +2244,7 @@ static ssize_t n_tty_read(struct tty_struct *tty, struct file *file, n_tty_check_unthrottle(tty); - if (b - buf >= minimum) + if (kb - kbuf >= minimum) break; if (time) timeout = time; @@ -2270,8 +2256,8 @@ static ssize_t n_tty_read(struct tty_struct *tty, struct file *file, remove_wait_queue(&tty->read_wait, &wait); mutex_unlock(&ldata->atomic_read_lock); - if (b - buf) - retval = b - buf; + if (kb - kbuf) + retval = kb - kbuf; return retval; } diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 338bc4ef5549..a34f8bcf875e 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -833,6 +833,65 @@ static void tty_update_time(struct timespec64 *time) time->tv_sec = sec; } +/* + * Iterate on the ldisc ->read() function until we've gotten all + * the data the ldisc has for us. + * + * The "cookie" is something that the ldisc read function can fill + * in to let us know that there is more data to be had. + * + * We promise to continue to call the ldisc until it stops returning + * data or clears the cookie. The cookie may be something that the + * ldisc maintains state for and needs to free. + */ +static int iterate_tty_read(struct tty_ldisc *ld, struct tty_struct *tty, struct file *file, + char __user *buf, size_t count) +{ + int retval = 0; + void *cookie = NULL; + unsigned long offset = 0; + char kernel_buf[64]; + + do { + int size, uncopied; + + size = count > sizeof(kernel_buf) ? sizeof(kernel_buf) : count; + size = ld->ops->read(tty, file, kernel_buf, size, &cookie, offset); + if (!size) + break; + + /* + * A ldisc read error return will override any previously copied + * data (eg -EOVERFLOW from HDLC) + */ + if (size < 0) { + memzero_explicit(kernel_buf, sizeof(kernel_buf)); + return size; + } + + uncopied = copy_to_user(buf+offset, kernel_buf, size); + size -= uncopied; + offset += size; + count -= size; + + /* + * If the user copy failed, we still need to do another ->read() + * call if we had a cookie to let the ldisc clear up. + * + * But make sure size is zeroed. + */ + if (unlikely(uncopied)) { + count = 0; + retval = -EFAULT; + } + } while (cookie); + + /* We always clear tty buffer in case they contained passwords */ + memzero_explicit(kernel_buf, sizeof(kernel_buf)); + return offset ? offset : retval; +} + + /** * tty_read - read method for tty device files * @file: pointer to tty file @@ -866,10 +925,9 @@ static ssize_t tty_read(struct file *file, char __user *buf, size_t count, ld = tty_ldisc_ref_wait(tty); if (!ld) return hung_up_tty_read(file, buf, count, ppos); + i = -EIO; if (ld->ops->read) - i = ld->ops->read(tty, file, buf, count); - else - i = -EIO; + i = iterate_tty_read(ld, tty, file, buf, count); tty_ldisc_deref(ld); if (i > 0) diff --git a/include/linux/tty_ldisc.h b/include/linux/tty_ldisc.h index b1e6043e9917..572a07976116 100644 --- a/include/linux/tty_ldisc.h +++ b/include/linux/tty_ldisc.h @@ -185,7 +185,8 @@ struct tty_ldisc_ops { void (*close)(struct tty_struct *); void (*flush_buffer)(struct tty_struct *tty); ssize_t (*read)(struct tty_struct *tty, struct file *file, - unsigned char __user *buf, size_t nr); + unsigned char *buf, size_t nr, + void **cookie, unsigned long offset); ssize_t (*write)(struct tty_struct *tty, struct file *file, const unsigned char *buf, size_t nr); int (*ioctl)(struct tty_struct *tty, struct file *file, diff --git a/net/nfc/nci/uart.c b/net/nfc/nci/uart.c index 11b554ce07ff..1204c438e87d 100644 --- a/net/nfc/nci/uart.c +++ b/net/nfc/nci/uart.c @@ -292,7 +292,8 @@ static int nci_uart_tty_ioctl(struct tty_struct *tty, struct file *file, /* We don't provide read/write/poll interface for user space. */ static ssize_t nci_uart_tty_read(struct tty_struct *tty, struct file *file, - unsigned char __user *buf, size_t nr) + unsigned char *buf, size_t nr, + void **cookie, unsigned long offset) { return 0; } -- cgit v1.2.3 From 7baf2429a1a965369b0ce44efb6315cdd515aa9c Mon Sep 17 00:00:00 2001 From: wenxu Date: Tue, 19 Jan 2021 16:31:50 +0800 Subject: net/sched: cls_flower add CT_FLAGS_INVALID flag support This patch add the TCA_FLOWER_KEY_CT_FLAGS_INVALID flag to match the ct_state with invalid for conntrack. Signed-off-by: wenxu Acked-by: Marcelo Ricardo Leitner Link: https://lore.kernel.org/r/1611045110-682-1-git-send-email-wenxu@ucloud.cn Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 4 ++-- include/net/sch_generic.h | 1 + include/uapi/linux/pkt_cls.h | 1 + net/core/dev.c | 2 ++ net/core/flow_dissector.c | 13 +++++++++---- net/sched/act_ct.c | 1 + net/sched/cls_flower.c | 4 +++- 7 files changed, 19 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 46f901adf1a8..186dad231e30 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1353,8 +1353,8 @@ void skb_flow_dissect_ct(const struct sk_buff *skb, struct flow_dissector *flow_dissector, void *target_container, - u16 *ctinfo_map, - size_t mapsize); + u16 *ctinfo_map, size_t mapsize, + bool post_ct); void skb_flow_dissect_tunnel_info(const struct sk_buff *skb, struct flow_dissector *flow_dissector, diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 639e465a108f..e7bee99aebce 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -388,6 +388,7 @@ struct qdisc_skb_cb { #define QDISC_CB_PRIV_LEN 20 unsigned char data[QDISC_CB_PRIV_LEN]; u16 mru; + bool post_ct; }; typedef void tcf_chain_head_change_t(struct tcf_proto *tp_head, void *priv); diff --git a/include/uapi/linux/pkt_cls.h b/include/uapi/linux/pkt_cls.h index ee95f42fb0ec..709668e264b0 100644 --- a/include/uapi/linux/pkt_cls.h +++ b/include/uapi/linux/pkt_cls.h @@ -591,6 +591,7 @@ enum { TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED = 1 << 1, /* Part of an existing connection. */ TCA_FLOWER_KEY_CT_FLAGS_RELATED = 1 << 2, /* Related to an established connection. */ TCA_FLOWER_KEY_CT_FLAGS_TRACKED = 1 << 3, /* Conntrack has occurred. */ + TCA_FLOWER_KEY_CT_FLAGS_INVALID = 1 << 4, /* Conntrack is invalid. */ }; enum { diff --git a/net/core/dev.c b/net/core/dev.c index 00f970ba0248..d9ce02e95992 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3878,6 +3878,7 @@ sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */ qdisc_skb_cb(skb)->mru = 0; + qdisc_skb_cb(skb)->post_ct = false; mini_qdisc_bstats_cpu_update(miniq, skb); switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) { @@ -4960,6 +4961,7 @@ sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, qdisc_skb_cb(skb)->pkt_len = skb->len; qdisc_skb_cb(skb)->mru = 0; + qdisc_skb_cb(skb)->post_ct = false; skb->tc_at_ingress = 1; mini_qdisc_bstats_cpu_update(miniq, skb); diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 2d70ded389ae..c565c7a17091 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -237,9 +237,8 @@ skb_flow_dissect_set_enc_addr_type(enum flow_dissector_key_id type, void skb_flow_dissect_ct(const struct sk_buff *skb, struct flow_dissector *flow_dissector, - void *target_container, - u16 *ctinfo_map, - size_t mapsize) + void *target_container, u16 *ctinfo_map, + size_t mapsize, bool post_ct) { #if IS_ENABLED(CONFIG_NF_CONNTRACK) struct flow_dissector_key_ct *key; @@ -251,13 +250,19 @@ skb_flow_dissect_ct(const struct sk_buff *skb, return; ct = nf_ct_get(skb, &ctinfo); - if (!ct) + if (!ct && !post_ct) return; key = skb_flow_dissector_target(flow_dissector, FLOW_DISSECTOR_KEY_CT, target_container); + if (!ct) { + key->ct_state = TCA_FLOWER_KEY_CT_FLAGS_TRACKED | + TCA_FLOWER_KEY_CT_FLAGS_INVALID; + return; + } + if (ctinfo < mapsize) key->ct_state = ctinfo_map[ctinfo]; #if IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index 83a5c6722a06..b3442078aabc 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -1030,6 +1030,7 @@ out_push: out: tcf_action_update_bstats(&c->common, skb); + qdisc_skb_cb(skb)->post_ct = true; if (defrag) qdisc_skb_cb(skb)->pkt_len = skb->len; return retval; diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c index 84f932532db7..4a9297a89c77 100644 --- a/net/sched/cls_flower.c +++ b/net/sched/cls_flower.c @@ -302,6 +302,7 @@ static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp, struct tcf_result *res) { struct cls_fl_head *head = rcu_dereference_bh(tp->root); + bool post_ct = qdisc_skb_cb(skb)->post_ct; struct fl_flow_key skb_key; struct fl_flow_mask *mask; struct cls_fl_filter *f; @@ -318,7 +319,8 @@ static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp, skb_flow_dissect_tunnel_info(skb, &mask->dissector, &skb_key); skb_flow_dissect_ct(skb, &mask->dissector, &skb_key, fl_ct_info_to_flower_map, - ARRAY_SIZE(fl_ct_info_to_flower_map)); + ARRAY_SIZE(fl_ct_info_to_flower_map), + post_ct); skb_flow_dissect_hash(skb, &mask->dissector, &skb_key); skb_flow_dissect(skb, &mask->dissector, &skb_key, 0); -- cgit v1.2.3 From 0908c5aca31eb5e0c72d7a5dba422629b88e877d Mon Sep 17 00:00:00 2001 From: Kyle Tso Date: Thu, 14 Jan 2021 22:50:51 +0800 Subject: usb: typec: tcpm: AMS and Collision Avoidance This patch provides the implementation of Collision Avoidance introduced in PD3.0. The start of each Atomic Message Sequence (AMS) initiated by the port will be denied if the current AMS is not interruptible. The Source port will set the CC to SinkTxNG if it is going to initiate an AMS, and SinkTxOk otherwise. Meanwhile, any AMS initiated by a Sink port will be denied in TCPM if the port partner (Source) sets SinkTxNG except for HARD_RESET and SOFT_RESET. Tested-by: Hans de Goede Reviewed-by: Heikki Krogerus Signed-off-by: Kyle Tso Link: https://lore.kernel.org/r/20210114145053.1952756-2-kyletso@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tcpm/tcpm.c | 620 +++++++++++++++++++++++++++++++++++------- include/linux/usb/pd.h | 1 + include/linux/usb/tcpm.h | 4 + 3 files changed, 527 insertions(+), 98 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 22a85b396f69..2b16d2764092 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -76,6 +76,8 @@ S(SNK_HARD_RESET_SINK_ON), \ \ S(SOFT_RESET), \ + S(SRC_SOFT_RESET_WAIT_SNK_TX), \ + S(SNK_SOFT_RESET), \ S(SOFT_RESET_SEND), \ \ S(DR_SWAP_ACCEPT), \ @@ -139,7 +141,45 @@ \ S(ERROR_RECOVERY), \ S(PORT_RESET), \ - S(PORT_RESET_WAIT_OFF) + S(PORT_RESET_WAIT_OFF), \ + \ + S(AMS_START) + +#define FOREACH_AMS(S) \ + S(NONE_AMS), \ + S(POWER_NEGOTIATION), \ + S(GOTOMIN), \ + S(SOFT_RESET_AMS), \ + S(HARD_RESET), \ + S(CABLE_RESET), \ + S(GET_SOURCE_CAPABILITIES), \ + S(GET_SINK_CAPABILITIES), \ + S(POWER_ROLE_SWAP), \ + S(FAST_ROLE_SWAP), \ + S(DATA_ROLE_SWAP), \ + S(VCONN_SWAP), \ + S(SOURCE_ALERT), \ + S(GETTING_SOURCE_EXTENDED_CAPABILITIES),\ + S(GETTING_SOURCE_SINK_STATUS), \ + S(GETTING_BATTERY_CAPABILITIES), \ + S(GETTING_BATTERY_STATUS), \ + S(GETTING_MANUFACTURER_INFORMATION), \ + S(SECURITY), \ + S(FIRMWARE_UPDATE), \ + S(DISCOVER_IDENTITY), \ + S(SOURCE_STARTUP_CABLE_PLUG_DISCOVER_IDENTITY), \ + S(DISCOVER_SVIDS), \ + S(DISCOVER_MODES), \ + S(DFP_TO_UFP_ENTER_MODE), \ + S(DFP_TO_UFP_EXIT_MODE), \ + S(DFP_TO_CABLE_PLUG_ENTER_MODE), \ + S(DFP_TO_CABLE_PLUG_EXIT_MODE), \ + S(ATTENTION), \ + S(BIST), \ + S(UNSTRUCTURED_VDMS), \ + S(STRUCTURED_VDMS), \ + S(COUNTRY_INFO), \ + S(COUNTRY_CODES) #define GENERATE_ENUM(e) e #define GENERATE_STRING(s) #s @@ -152,6 +192,14 @@ static const char * const tcpm_states[] = { FOREACH_STATE(GENERATE_STRING) }; +enum tcpm_ams { + FOREACH_AMS(GENERATE_ENUM) +}; + +static const char * const tcpm_ams_str[] = { + FOREACH_AMS(GENERATE_STRING) +}; + enum vdm_states { VDM_STATE_ERR_BUSY = -3, VDM_STATE_ERR_SEND = -2, @@ -161,6 +209,7 @@ enum vdm_states { VDM_STATE_READY = 1, VDM_STATE_BUSY = 2, VDM_STATE_WAIT_RSP_BUSY = 3, + VDM_STATE_SEND_MESSAGE = 4, }; enum pd_msg_request { @@ -381,6 +430,11 @@ struct tcpm_port { /* Sink caps have been queried */ bool sink_cap_done; + /* Collision Avoidance and Atomic Message Sequence */ + enum tcpm_state upcoming_state; + enum tcpm_ams ams; + bool in_ams; + #ifdef CONFIG_DEBUG_FS struct dentry *dentry; struct mutex logbuffer_lock; /* log buffer access lock */ @@ -396,6 +450,12 @@ struct pd_rx_event { struct pd_message msg; }; +static const char * const pd_rev[] = { + [PD_REV10] = "rev1", + [PD_REV20] = "rev2", + [PD_REV30] = "rev3", +}; + #define tcpm_cc_is_sink(cc) \ ((cc) == TYPEC_CC_RP_DEF || (cc) == TYPEC_CC_RP_1_5 || \ (cc) == TYPEC_CC_RP_3_0) @@ -440,6 +500,10 @@ struct pd_rx_event { ((port)->typec_caps.data == TYPEC_PORT_DFP ? \ TYPEC_HOST : TYPEC_DEVICE) +#define tcpm_sink_tx_ok(port) \ + (tcpm_port_is_sink(port) && \ + ((port)->cc1 == TYPEC_CC_RP_3_0 || (port)->cc2 == TYPEC_CC_RP_3_0)) + static enum tcpm_state tcpm_default_state(struct tcpm_port *port) { if (port->port_type == TYPEC_PORT_DRP) { @@ -666,6 +730,67 @@ static void tcpm_debugfs_exit(const struct tcpm_port *port) { } #endif +static void tcpm_set_cc(struct tcpm_port *port, enum typec_cc_status cc) +{ + tcpm_log(port, "cc:=%d", cc); + port->cc_req = cc; + port->tcpc->set_cc(port->tcpc, cc); +} + +/* + * Determine RP value to set based on maximum current supported + * by a port if configured as source. + * Returns CC value to report to link partner. + */ +static enum typec_cc_status tcpm_rp_cc(struct tcpm_port *port) +{ + const u32 *src_pdo = port->src_pdo; + int nr_pdo = port->nr_src_pdo; + int i; + + /* + * Search for first entry with matching voltage. + * It should report the maximum supported current. + */ + for (i = 0; i < nr_pdo; i++) { + const u32 pdo = src_pdo[i]; + + if (pdo_type(pdo) == PDO_TYPE_FIXED && + pdo_fixed_voltage(pdo) == 5000) { + unsigned int curr = pdo_max_current(pdo); + + if (curr >= 3000) + return TYPEC_CC_RP_3_0; + else if (curr >= 1500) + return TYPEC_CC_RP_1_5; + return TYPEC_CC_RP_DEF; + } + } + + return TYPEC_CC_RP_DEF; +} + +static int tcpm_ams_finish(struct tcpm_port *port) +{ + int ret = 0; + + tcpm_log(port, "AMS %s finished", tcpm_ams_str[port->ams]); + + if (port->pd_capable && port->pwr_role == TYPEC_SOURCE) { + if (port->negotiated_rev >= PD_REV30) + tcpm_set_cc(port, SINK_TX_OK); + else + tcpm_set_cc(port, SINK_TX_NG); + } else if (port->pwr_role == TYPEC_SOURCE) { + tcpm_set_cc(port, tcpm_rp_cc(port)); + } + + port->in_ams = false; + port->ams = NONE_AMS; + + return ret; +} + static int tcpm_pd_transmit(struct tcpm_port *port, enum tcpm_transmit_type type, const struct pd_message *msg) @@ -693,13 +818,30 @@ static int tcpm_pd_transmit(struct tcpm_port *port, switch (port->tx_status) { case TCPC_TX_SUCCESS: port->message_id = (port->message_id + 1) & PD_HEADER_ID_MASK; - return 0; + /* + * USB PD rev 2.0, 8.3.2.2.1: + * USB PD rev 3.0, 8.3.2.1.3: + * "... Note that every AMS is Interruptible until the first + * Message in the sequence has been successfully sent (GoodCRC + * Message received)." + */ + if (port->ams != NONE_AMS) + port->in_ams = true; + break; case TCPC_TX_DISCARDED: - return -EAGAIN; + ret = -EAGAIN; + break; case TCPC_TX_FAILED: default: - return -EIO; + ret = -EIO; + break; } + + /* Some AMS don't expect responses. Finish them here. */ + if (port->ams == ATTENTION || port->ams == SOURCE_ALERT) + tcpm_ams_finish(port); + + return ret; } void tcpm_pd_transmit_complete(struct tcpm_port *port, @@ -804,39 +946,6 @@ static int tcpm_set_current_limit(struct tcpm_port *port, u32 max_ma, u32 mv) return ret; } -/* - * Determine RP value to set based on maximum current supported - * by a port if configured as source. - * Returns CC value to report to link partner. - */ -static enum typec_cc_status tcpm_rp_cc(struct tcpm_port *port) -{ - const u32 *src_pdo = port->src_pdo; - int nr_pdo = port->nr_src_pdo; - int i; - - /* - * Search for first entry with matching voltage. - * It should report the maximum supported current. - */ - for (i = 0; i < nr_pdo; i++) { - const u32 pdo = src_pdo[i]; - - if (pdo_type(pdo) == PDO_TYPE_FIXED && - pdo_fixed_voltage(pdo) == 5000) { - unsigned int curr = pdo_max_current(pdo); - - if (curr >= 3000) - return TYPEC_CC_RP_3_0; - else if (curr >= 1500) - return TYPEC_CC_RP_1_5; - return TYPEC_CC_RP_DEF; - } - } - - return TYPEC_CC_RP_DEF; -} - static int tcpm_set_attached_state(struct tcpm_port *port, bool attached) { return port->tcpc->set_roles(port->tcpc, attached, port->pwr_role, @@ -1000,16 +1109,17 @@ static void tcpm_set_state(struct tcpm_port *port, enum tcpm_state state, unsigned int delay_ms) { if (delay_ms) { - tcpm_log(port, "pending state change %s -> %s @ %u ms", - tcpm_states[port->state], tcpm_states[state], - delay_ms); + tcpm_log(port, "pending state change %s -> %s @ %u ms [%s %s]", + tcpm_states[port->state], tcpm_states[state], delay_ms, + pd_rev[port->negotiated_rev], tcpm_ams_str[port->ams]); port->delayed_state = state; mod_tcpm_delayed_work(port, delay_ms); port->delayed_runtime = ktime_add(ktime_get(), ms_to_ktime(delay_ms)); port->delay_ms = delay_ms; } else { - tcpm_log(port, "state change %s -> %s", - tcpm_states[port->state], tcpm_states[state]); + tcpm_log(port, "state change %s -> %s [%s %s]", + tcpm_states[port->state], tcpm_states[state], + pd_rev[port->negotiated_rev], tcpm_ams_str[port->ams]); port->delayed_state = INVALID_STATE; port->prev_state = port->state; port->state = state; @@ -1031,10 +1141,11 @@ static void tcpm_set_state_cond(struct tcpm_port *port, enum tcpm_state state, tcpm_set_state(port, state, delay_ms); else tcpm_log(port, - "skipped %sstate change %s -> %s [%u ms], context state %s", + "skipped %sstate change %s -> %s [%u ms], context state %s [%s %s]", delay_ms ? "delayed " : "", tcpm_states[port->state], tcpm_states[state], - delay_ms, tcpm_states[port->enter_state]); + delay_ms, tcpm_states[port->enter_state], + pd_rev[port->negotiated_rev], tcpm_ams_str[port->ams]); } static void tcpm_queue_message(struct tcpm_port *port, @@ -1044,6 +1155,149 @@ static void tcpm_queue_message(struct tcpm_port *port, mod_tcpm_delayed_work(port, 0); } +static bool tcpm_vdm_ams(struct tcpm_port *port) +{ + switch (port->ams) { + case DISCOVER_IDENTITY: + case SOURCE_STARTUP_CABLE_PLUG_DISCOVER_IDENTITY: + case DISCOVER_SVIDS: + case DISCOVER_MODES: + case DFP_TO_UFP_ENTER_MODE: + case DFP_TO_UFP_EXIT_MODE: + case DFP_TO_CABLE_PLUG_ENTER_MODE: + case DFP_TO_CABLE_PLUG_EXIT_MODE: + case ATTENTION: + case UNSTRUCTURED_VDMS: + case STRUCTURED_VDMS: + break; + default: + return false; + } + + return true; +} + +static bool tcpm_ams_interruptible(struct tcpm_port *port) +{ + switch (port->ams) { + /* Interruptible AMS */ + case NONE_AMS: + case SECURITY: + case FIRMWARE_UPDATE: + case DISCOVER_IDENTITY: + case SOURCE_STARTUP_CABLE_PLUG_DISCOVER_IDENTITY: + case DISCOVER_SVIDS: + case DISCOVER_MODES: + case DFP_TO_UFP_ENTER_MODE: + case DFP_TO_UFP_EXIT_MODE: + case DFP_TO_CABLE_PLUG_ENTER_MODE: + case DFP_TO_CABLE_PLUG_EXIT_MODE: + case UNSTRUCTURED_VDMS: + case STRUCTURED_VDMS: + case COUNTRY_INFO: + case COUNTRY_CODES: + break; + /* Non-Interruptible AMS */ + default: + if (port->in_ams) + return false; + break; + } + + return true; +} + +static int tcpm_ams_start(struct tcpm_port *port, enum tcpm_ams ams) +{ + int ret = 0; + + tcpm_log(port, "AMS %s start", tcpm_ams_str[ams]); + + if (!tcpm_ams_interruptible(port) && ams != HARD_RESET) { + port->upcoming_state = INVALID_STATE; + tcpm_log(port, "AMS %s not interruptible, aborting", + tcpm_ams_str[port->ams]); + return -EAGAIN; + } + + if (port->pwr_role == TYPEC_SOURCE) { + enum typec_cc_status cc_req = port->cc_req; + + port->ams = ams; + + if (ams == HARD_RESET) { + tcpm_set_cc(port, tcpm_rp_cc(port)); + tcpm_pd_transmit(port, TCPC_TX_HARD_RESET, NULL); + tcpm_set_state(port, HARD_RESET_START, 0); + return ret; + } else if (ams == SOFT_RESET_AMS) { + if (!port->explicit_contract) { + port->upcoming_state = INVALID_STATE; + tcpm_set_cc(port, tcpm_rp_cc(port)); + return ret; + } + } else if (tcpm_vdm_ams(port)) { + /* tSinkTx is enforced in vdm_run_state_machine */ + if (port->negotiated_rev >= PD_REV30) + tcpm_set_cc(port, SINK_TX_NG); + return ret; + } + + if (port->negotiated_rev >= PD_REV30) + tcpm_set_cc(port, SINK_TX_NG); + + switch (port->state) { + case SRC_READY: + case SRC_STARTUP: + case SRC_SOFT_RESET_WAIT_SNK_TX: + case SOFT_RESET: + case SOFT_RESET_SEND: + if (port->negotiated_rev >= PD_REV30) + tcpm_set_state(port, AMS_START, + cc_req == SINK_TX_OK ? + PD_T_SINK_TX : 0); + else + tcpm_set_state(port, AMS_START, 0); + break; + default: + if (port->negotiated_rev >= PD_REV30) + tcpm_set_state(port, SRC_READY, + cc_req == SINK_TX_OK ? + PD_T_SINK_TX : 0); + else + tcpm_set_state(port, SRC_READY, 0); + break; + } + } else { + if (port->negotiated_rev >= PD_REV30 && + !tcpm_sink_tx_ok(port) && + ams != SOFT_RESET_AMS && + ams != HARD_RESET) { + port->upcoming_state = INVALID_STATE; + tcpm_log(port, "Sink TX No Go"); + return -EAGAIN; + } + + port->ams = ams; + + if (ams == HARD_RESET) { + tcpm_pd_transmit(port, TCPC_TX_HARD_RESET, NULL); + tcpm_set_state(port, HARD_RESET_START, 0); + return ret; + } else if (tcpm_vdm_ams(port)) { + return ret; + } + + if (port->state == SNK_READY || + port->state == SNK_SOFT_RESET) + tcpm_set_state(port, AMS_START, 0); + else + tcpm_set_state(port, SNK_READY, 0); + } + + return ret; +} + /* * VDM/VDO handling functions */ @@ -1236,6 +1490,8 @@ static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev, if (IS_ERR_OR_NULL(port->partner)) break; + tcpm_ams_finish(port); + switch (cmd) { case CMD_DISCOVER_IDENT: /* 6.4.4.3.1 */ @@ -1286,6 +1542,7 @@ static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev, } break; case CMDT_RSP_NAK: + tcpm_ams_finish(port); switch (cmd) { case CMD_ENTER_MODE: /* Back to USB Operation */ @@ -1435,7 +1692,8 @@ static unsigned int vdm_ready_timeout(u32 vdm_hdr) static void vdm_run_state_machine(struct tcpm_port *port) { struct pd_message msg; - int i, res; + int i, res = 0; + u32 vdo_hdr = port->vdo_data[0]; switch (port->vdm_state) { case VDM_STATE_READY: @@ -1452,26 +1710,45 @@ static void vdm_run_state_machine(struct tcpm_port *port) if (port->state != SRC_READY && port->state != SNK_READY) break; - /* Prepare and send VDM */ - memset(&msg, 0, sizeof(msg)); - msg.header = PD_HEADER_LE(PD_DATA_VENDOR_DEF, - port->pwr_role, - port->data_role, - port->negotiated_rev, - port->message_id, port->vdo_count); - for (i = 0; i < port->vdo_count; i++) - msg.payload[i] = cpu_to_le32(port->vdo_data[i]); - res = tcpm_pd_transmit(port, TCPC_TX_SOP, &msg); - if (res < 0) { - port->vdm_state = VDM_STATE_ERR_SEND; - } else { - unsigned long timeout; + /* TODO: AMS operation for Unstructured VDM */ + if (PD_VDO_SVDM(vdo_hdr) && PD_VDO_CMDT(vdo_hdr) == CMDT_INIT) { + switch (PD_VDO_CMD(vdo_hdr)) { + case CMD_DISCOVER_IDENT: + res = tcpm_ams_start(port, DISCOVER_IDENTITY); + break; + case CMD_DISCOVER_SVID: + res = tcpm_ams_start(port, DISCOVER_SVIDS); + break; + case CMD_DISCOVER_MODES: + res = tcpm_ams_start(port, DISCOVER_MODES); + break; + case CMD_ENTER_MODE: + res = tcpm_ams_start(port, DFP_TO_UFP_ENTER_MODE); + break; + case CMD_EXIT_MODE: + res = tcpm_ams_start(port, DFP_TO_UFP_EXIT_MODE); + break; + case CMD_ATTENTION: + res = tcpm_ams_start(port, ATTENTION); + break; + case VDO_CMD_VENDOR(0) ... VDO_CMD_VENDOR(15): + res = tcpm_ams_start(port, STRUCTURED_VDMS); + break; + default: + res = -EOPNOTSUPP; + break; + } - port->vdm_retries = 0; - port->vdm_state = VDM_STATE_BUSY; - timeout = vdm_ready_timeout(port->vdo_data[0]); - mod_vdm_delayed_work(port, timeout); + if (res < 0) + return; } + + port->vdm_state = VDM_STATE_SEND_MESSAGE; + mod_vdm_delayed_work(port, (port->negotiated_rev >= PD_REV30 && + port->pwr_role == TYPEC_SOURCE && + PD_VDO_SVDM(vdo_hdr) && + PD_VDO_CMDT(vdo_hdr) == CMDT_INIT) ? + PD_T_SINK_TX : 0); break; case VDM_STATE_WAIT_RSP_BUSY: port->vdo_data[0] = port->vdo_retry; @@ -1480,6 +1757,8 @@ static void vdm_run_state_machine(struct tcpm_port *port) break; case VDM_STATE_BUSY: port->vdm_state = VDM_STATE_ERR_TMOUT; + if (port->ams != NONE_AMS) + tcpm_ams_finish(port); break; case VDM_STATE_ERR_SEND: /* @@ -1492,6 +1771,29 @@ static void vdm_run_state_machine(struct tcpm_port *port) tcpm_log(port, "VDM Tx error, retry"); port->vdm_retries++; port->vdm_state = VDM_STATE_READY; + tcpm_ams_finish(port); + } + break; + case VDM_STATE_SEND_MESSAGE: + /* Prepare and send VDM */ + memset(&msg, 0, sizeof(msg)); + msg.header = PD_HEADER_LE(PD_DATA_VENDOR_DEF, + port->pwr_role, + port->data_role, + port->negotiated_rev, + port->message_id, port->vdo_count); + for (i = 0; i < port->vdo_count; i++) + msg.payload[i] = cpu_to_le32(port->vdo_data[i]); + res = tcpm_pd_transmit(port, TCPC_TX_SOP, &msg); + if (res < 0) { + port->vdm_state = VDM_STATE_ERR_SEND; + } else { + unsigned long timeout; + + port->vdm_retries = 0; + port->vdm_state = VDM_STATE_BUSY; + timeout = vdm_ready_timeout(vdo_hdr); + mod_vdm_delayed_work(port, timeout); } break; default: @@ -1514,7 +1816,8 @@ static void vdm_state_machine_work(struct kthread_work *work) prev_state = port->vdm_state; vdm_run_state_machine(port); } while (port->vdm_state != prev_state && - port->vdm_state != VDM_STATE_BUSY); + port->vdm_state != VDM_STATE_BUSY && + port->vdm_state != VDM_STATE_SEND_MESSAGE); mutex_unlock(&port->lock); } @@ -1997,11 +2300,14 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port, case SOFT_RESET_SEND: port->message_id = 0; port->rx_msgid = -1; - if (port->pwr_role == TYPEC_SOURCE) - next_state = SRC_SEND_CAPABILITIES; - else - next_state = SNK_WAIT_CAPABILITIES; - tcpm_set_state(port, next_state, 0); + if (port->ams == SOFT_RESET_AMS) + tcpm_ams_finish(port); + if (port->pwr_role == TYPEC_SOURCE) { + port->upcoming_state = SRC_SEND_CAPABILITIES; + tcpm_ams_start(port, POWER_NEGOTIATION); + } else { + tcpm_set_state(port, SNK_WAIT_CAPABILITIES, 0); + } break; case DR_SWAP_SEND: tcpm_set_state(port, DR_SWAP_CHANGE_DR, 0); @@ -2776,13 +3082,6 @@ static bool tcpm_start_toggling(struct tcpm_port *port, enum typec_cc_status cc) return ret == 0; } -static void tcpm_set_cc(struct tcpm_port *port, enum typec_cc_status cc) -{ - tcpm_log(port, "cc:=%d", cc); - port->cc_req = cc; - port->tcpc->set_cc(port->tcpc, cc); -} - static int tcpm_init_vbus(struct tcpm_port *port) { int ret; @@ -2912,6 +3211,8 @@ static void tcpm_reset_port(struct tcpm_port *port) ret = port->tcpc->enable_auto_vbus_discharge(port->tcpc, false); tcpm_log_force(port, "Disable vbus discharge ret:%d", ret); } + port->in_ams = false; + port->ams = NONE_AMS; tcpm_unregister_altmodes(port); tcpm_typec_disconnect(port); port->attached = false; @@ -3090,6 +3391,7 @@ static void run_state_machine(struct tcpm_port *port) int ret; enum typec_pwr_opmode opmode; unsigned int msecs; + enum tcpm_state upcoming_state; port->enter_state = port->state; switch (port->state) { @@ -3190,7 +3492,12 @@ static void run_state_machine(struct tcpm_port *port) port->message_id = 0; port->rx_msgid = -1; port->explicit_contract = false; - tcpm_set_state(port, SRC_SEND_CAPABILITIES, 0); + /* SNK -> SRC POWER/FAST_ROLE_SWAP finished */ + if (port->ams == POWER_ROLE_SWAP || + port->ams == FAST_ROLE_SWAP) + tcpm_ams_finish(port); + port->upcoming_state = SRC_SEND_CAPABILITIES; + tcpm_ams_start(port, POWER_NEGOTIATION); break; case SRC_SEND_CAPABILITIES: port->caps_count++; @@ -3272,6 +3579,19 @@ static void run_state_machine(struct tcpm_port *port) tcpm_swap_complete(port, 0); tcpm_typec_connect(port); + if (port->ams != NONE_AMS) + tcpm_ams_finish(port); + /* + * If previous AMS is interrupted, switch to the upcoming + * state. + */ + if (port->upcoming_state != INVALID_STATE) { + upcoming_state = port->upcoming_state; + port->upcoming_state = INVALID_STATE; + tcpm_set_state(port, upcoming_state, 0); + break; + } + tcpm_check_send_discover(port); /* * 6.3.5 @@ -3389,6 +3709,12 @@ static void run_state_machine(struct tcpm_port *port) port->message_id = 0; port->rx_msgid = -1; port->explicit_contract = false; + + if (port->ams == POWER_ROLE_SWAP || + port->ams == FAST_ROLE_SWAP) + /* SRC -> SNK POWER/FAST_ROLE_SWAP finished */ + tcpm_ams_finish(port); + tcpm_set_state(port, SNK_DISCOVERY, 0); break; case SNK_DISCOVERY: @@ -3437,7 +3763,7 @@ static void run_state_machine(struct tcpm_port *port) */ if (port->vbus_never_low) { port->vbus_never_low = false; - tcpm_set_state(port, SOFT_RESET_SEND, + tcpm_set_state(port, SNK_SOFT_RESET, PD_T_SINK_WAIT_CAP); } else { tcpm_set_state(port, hard_reset_state(port), @@ -3490,9 +3816,23 @@ static void run_state_machine(struct tcpm_port *port) tcpm_swap_complete(port, 0); tcpm_typec_connect(port); - tcpm_check_send_discover(port); mod_enable_frs_delayed_work(port, 0); tcpm_pps_complete(port, port->pps_status); + + if (port->ams != NONE_AMS) + tcpm_ams_finish(port); + /* + * If previous AMS is interrupted, switch to the upcoming + * state. + */ + if (port->upcoming_state != INVALID_STATE) { + upcoming_state = port->upcoming_state; + port->upcoming_state = INVALID_STATE; + tcpm_set_state(port, upcoming_state, 0); + break; + } + + tcpm_check_send_discover(port); power_supply_changed(port->psy); break; @@ -3513,8 +3853,14 @@ static void run_state_machine(struct tcpm_port *port) /* Hard_Reset states */ case HARD_RESET_SEND: - tcpm_pd_transmit(port, TCPC_TX_HARD_RESET, NULL); - tcpm_set_state(port, HARD_RESET_START, 0); + if (port->ams != NONE_AMS) + tcpm_ams_finish(port); + /* + * State machine will be directed to HARD_RESET_START, + * thus set upcoming_state to INVALID_STATE. + */ + port->upcoming_state = INVALID_STATE; + tcpm_ams_start(port, HARD_RESET); break; case HARD_RESET_START: port->sink_cap_done = false; @@ -3558,6 +3904,8 @@ static void run_state_machine(struct tcpm_port *port) case SRC_HARD_RESET_VBUS_ON: tcpm_set_vconn(port, true); tcpm_set_vbus(port, true); + if (port->ams == HARD_RESET) + tcpm_ams_finish(port); port->tcpc->set_pd_rx(port->tcpc, true); tcpm_set_attached_state(port, true); tcpm_set_state(port, SRC_UNATTACHED, PD_T_PS_SOURCE_ON); @@ -3579,6 +3927,8 @@ static void run_state_machine(struct tcpm_port *port) tcpm_set_state(port, SNK_HARD_RESET_SINK_ON, PD_T_SAFE_0V); break; case SNK_HARD_RESET_WAIT_VBUS: + if (port->ams == HARD_RESET) + tcpm_ams_finish(port); /* Assume we're disconnected if VBUS doesn't come back. */ tcpm_set_state(port, SNK_UNATTACHED, PD_T_SRC_RECOVER_MAX + PD_T_SRC_TURN_ON); @@ -3606,6 +3956,8 @@ static void run_state_machine(struct tcpm_port *port) 5000); tcpm_set_charge(port, true); } + if (port->ams == HARD_RESET) + tcpm_ams_finish(port); tcpm_set_attached_state(port, true); tcpm_set_auto_vbus_discharge_threshold(port, TYPEC_PWR_MODE_USB, false, VSAFE5V); tcpm_set_state(port, SNK_STARTUP, 0); @@ -3616,10 +3968,19 @@ static void run_state_machine(struct tcpm_port *port) port->message_id = 0; port->rx_msgid = -1; tcpm_pd_send_control(port, PD_CTRL_ACCEPT); - if (port->pwr_role == TYPEC_SOURCE) - tcpm_set_state(port, SRC_SEND_CAPABILITIES, 0); - else + if (port->pwr_role == TYPEC_SOURCE) { + port->upcoming_state = SRC_SEND_CAPABILITIES; + tcpm_ams_start(port, POWER_NEGOTIATION); + } else { tcpm_set_state(port, SNK_WAIT_CAPABILITIES, 0); + } + break; + case SRC_SOFT_RESET_WAIT_SNK_TX: + case SNK_SOFT_RESET: + if (port->ams != NONE_AMS) + tcpm_ams_finish(port); + port->upcoming_state = SOFT_RESET_SEND; + tcpm_ams_start(port, SOFT_RESET_AMS); break; case SOFT_RESET_SEND: port->message_id = 0; @@ -3886,6 +4247,19 @@ static void run_state_machine(struct tcpm_port *port) tcpm_default_state(port), port->vbus_present ? PD_T_PS_SOURCE_OFF : 0); break; + + /* AMS intermediate state */ + case AMS_START: + if (port->upcoming_state == INVALID_STATE) { + tcpm_set_state(port, port->pwr_role == TYPEC_SOURCE ? + SRC_READY : SNK_READY, 0); + break; + } + + upcoming_state = port->upcoming_state; + port->upcoming_state = INVALID_STATE; + tcpm_set_state(port, upcoming_state, 0); + break; default: WARN(1, "Unexpected port state %d\n", port->state); break; @@ -4313,6 +4687,8 @@ static void _tcpm_pd_hard_reset(struct tcpm_port *port) if (port->bist_request == BDO_MODE_TESTDATA && port->tcpc->set_bist_data) port->tcpc->set_bist_data(port->tcpc, false); + if (port->ams != NONE_AMS) + port->ams = NONE_AMS; /* * If we keep receiving hard reset requests, executing the hard reset * must have failed. Revert to error recovery if that happens. @@ -4363,10 +4739,16 @@ static void tcpm_pd_event_handler(struct kthread_work *work) _tcpm_cc_change(port, cc1, cc2); } if (events & TCPM_FRS_EVENT) { - if (port->state == SNK_READY) - tcpm_set_state(port, FR_SWAP_SEND, 0); - else + if (port->state == SNK_READY) { + int ret; + + port->upcoming_state = FR_SWAP_SEND; + ret = tcpm_ams_start(port, FAST_ROLE_SWAP); + if (ret == -EAGAIN) + port->upcoming_state = INVALID_STATE; + } else { tcpm_log(port, "Discarding FRS_SIGNAL! Not in sink ready"); + } } if (events & TCPM_SOURCING_VBUS) { tcpm_log(port, "sourcing vbus"); @@ -4435,6 +4817,7 @@ EXPORT_SYMBOL_GPL(tcpm_sourcing_vbus); static void tcpm_enable_frs_work(struct kthread_work *work) { struct tcpm_port *port = container_of(work, struct tcpm_port, enable_frs); + int ret; mutex_lock(&port->lock); /* Not FRS capable */ @@ -4449,9 +4832,14 @@ static void tcpm_enable_frs_work(struct kthread_work *work) if (port->state != SNK_READY || port->vdm_state != VDM_STATE_DONE || port->send_discover) goto resched; - tcpm_set_state(port, GET_SINK_CAP, 0); - port->sink_cap_done = true; - + port->upcoming_state = GET_SINK_CAP; + ret = tcpm_ams_start(port, GET_SINK_CAPABILITIES); + if (ret == -EAGAIN) { + port->upcoming_state = INVALID_STATE; + } else { + port->sink_cap_done = true; + goto unlock; + } resched: mod_enable_frs_delayed_work(port, GET_SINK_CAP_RETRY_MS); unlock: @@ -4501,7 +4889,12 @@ static int tcpm_dr_set(struct typec_port *p, enum typec_data_role data) port->non_pd_role_swap = true; tcpm_set_state(port, PORT_RESET, 0); } else { - tcpm_set_state(port, DR_SWAP_SEND, 0); + port->upcoming_state = DR_SWAP_SEND; + ret = tcpm_ams_start(port, DATA_ROLE_SWAP); + if (ret == -EAGAIN) { + port->upcoming_state = INVALID_STATE; + goto port_unlock; + } } port->swap_status = 0; @@ -4547,10 +4940,16 @@ static int tcpm_pr_set(struct typec_port *p, enum typec_role role) goto port_unlock; } + port->upcoming_state = PR_SWAP_SEND; + ret = tcpm_ams_start(port, POWER_ROLE_SWAP); + if (ret == -EAGAIN) { + port->upcoming_state = INVALID_STATE; + goto port_unlock; + } + port->swap_status = 0; port->swap_pending = true; reinit_completion(&port->swap_complete); - tcpm_set_state(port, PR_SWAP_SEND, 0); mutex_unlock(&port->lock); if (!wait_for_completion_timeout(&port->swap_complete, @@ -4586,10 +4985,16 @@ static int tcpm_vconn_set(struct typec_port *p, enum typec_role role) goto port_unlock; } + port->upcoming_state = VCONN_SWAP_SEND; + ret = tcpm_ams_start(port, VCONN_SWAP); + if (ret == -EAGAIN) { + port->upcoming_state = INVALID_STATE; + goto port_unlock; + } + port->swap_status = 0; port->swap_pending = true; reinit_completion(&port->swap_complete); - tcpm_set_state(port, VCONN_SWAP_SEND, 0); mutex_unlock(&port->lock); if (!wait_for_completion_timeout(&port->swap_complete, @@ -4654,6 +5059,13 @@ static int tcpm_pps_set_op_curr(struct tcpm_port *port, u16 op_curr) goto port_unlock; } + port->upcoming_state = SNK_NEGOTIATE_PPS_CAPABILITIES; + ret = tcpm_ams_start(port, POWER_NEGOTIATION); + if (ret == -EAGAIN) { + port->upcoming_state = INVALID_STATE; + goto port_unlock; + } + /* Round down operating current to align with PPS valid steps */ op_curr = op_curr - (op_curr % RDO_PROG_CURR_MA_STEP); @@ -4661,7 +5073,6 @@ static int tcpm_pps_set_op_curr(struct tcpm_port *port, u16 op_curr) port->pps_data.op_curr = op_curr; port->pps_status = 0; port->pps_pending = true; - tcpm_set_state(port, SNK_NEGOTIATE_PPS_CAPABILITIES, 0); mutex_unlock(&port->lock); if (!wait_for_completion_timeout(&port->pps_complete, @@ -4710,6 +5121,13 @@ static int tcpm_pps_set_out_volt(struct tcpm_port *port, u16 out_volt) goto port_unlock; } + port->upcoming_state = SNK_NEGOTIATE_PPS_CAPABILITIES; + ret = tcpm_ams_start(port, POWER_NEGOTIATION); + if (ret == -EAGAIN) { + port->upcoming_state = INVALID_STATE; + goto port_unlock; + } + /* Round down output voltage to align with PPS valid steps */ out_volt = out_volt - (out_volt % RDO_PROG_VOLT_MV_STEP); @@ -4717,7 +5135,6 @@ static int tcpm_pps_set_out_volt(struct tcpm_port *port, u16 out_volt) port->pps_data.out_volt = out_volt; port->pps_status = 0; port->pps_pending = true; - tcpm_set_state(port, SNK_NEGOTIATE_PPS_CAPABILITIES, 0); mutex_unlock(&port->lock); if (!wait_for_completion_timeout(&port->pps_complete, @@ -4757,6 +5174,16 @@ static int tcpm_pps_activate(struct tcpm_port *port, bool activate) goto port_unlock; } + if (activate) + port->upcoming_state = SNK_NEGOTIATE_PPS_CAPABILITIES; + else + port->upcoming_state = SNK_NEGOTIATE_CAPABILITIES; + ret = tcpm_ams_start(port, POWER_NEGOTIATION); + if (ret == -EAGAIN) { + port->upcoming_state = INVALID_STATE; + goto port_unlock; + } + reinit_completion(&port->pps_complete); port->pps_status = 0; port->pps_pending = true; @@ -4765,9 +5192,6 @@ static int tcpm_pps_activate(struct tcpm_port *port, bool activate) if (activate) { port->pps_data.out_volt = port->supply_voltage; port->pps_data.op_curr = port->current_limit; - tcpm_set_state(port, SNK_NEGOTIATE_PPS_CAPABILITIES, 0); - } else { - tcpm_set_state(port, SNK_NEGOTIATE_CAPABILITIES, 0); } mutex_unlock(&port->lock); diff --git a/include/linux/usb/pd.h b/include/linux/usb/pd.h index bb9a782e1411..79599b90ba55 100644 --- a/include/linux/usb/pd.h +++ b/include/linux/usb/pd.h @@ -479,6 +479,7 @@ static inline unsigned int rdo_max_power(u32 rdo) #define PD_T_NEWSRC 250 /* Maximum of 275ms */ #define PD_T_SWAP_SRC_START 20 /* Minimum of 20ms */ #define PD_T_BIST_CONT_MODE 50 /* 30 - 60 ms */ +#define PD_T_SINK_TX 16 /* 16 - 20 ms */ #define PD_T_DRP_TRY 100 /* 75 - 150 ms */ #define PD_T_DRP_TRYWAIT 600 /* 400 - 800 ms */ diff --git a/include/linux/usb/tcpm.h b/include/linux/usb/tcpm.h index f4a18427f5c4..3af99f85e8b9 100644 --- a/include/linux/usb/tcpm.h +++ b/include/linux/usb/tcpm.h @@ -19,6 +19,10 @@ enum typec_cc_status { TYPEC_CC_RP_3_0, }; +/* Collision Avoidance */ +#define SINK_TX_NG TYPEC_CC_RP_1_5 +#define SINK_TX_OK TYPEC_CC_RP_3_0 + enum typec_cc_polarity { TYPEC_POLARITY_CC1, TYPEC_POLARITY_CC2, -- cgit v1.2.3 From 8dea75e11380fc59656fe965766ac192a831455f Mon Sep 17 00:00:00 2001 From: Kyle Tso Date: Thu, 14 Jan 2021 22:50:52 +0800 Subject: usb: typec: tcpm: Protocol Error handling PD3.0 Spec 6.8.1 describes how to handle Protocol Error. There are general rules defined in Table 6-61 which regulate incoming Message handling. If the incoming Message is unexpected, unsupported, or unrecognized, Protocol Error occurs. Follow the rules to handle these situations. Also consider PD2.0 connection (PD2.0 Spec Table 6-36) for backward compatibilities. To know the types of AMS in all the recipient's states, identify those AMS who are initiated by the port partner but not yet recorded in the current code. Besides, introduce a new state CHUNK_NOT_SUPP to delay the NOT_SUPPORTED message after receiving a chunked message. Tested-by: Hans de Goede Reviewed-by: Heikki Krogerus Signed-off-by: Kyle Tso Link: https://lore.kernel.org/r/20210114145053.1952756-3-kyletso@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tcpm/tcpm.c | 335 ++++++++++++++++++++++++++++++------------ include/linux/usb/pd.h | 1 + 2 files changed, 246 insertions(+), 90 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 2b16d2764092..70922723da4b 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -143,7 +143,8 @@ S(PORT_RESET), \ S(PORT_RESET_WAIT_OFF), \ \ - S(AMS_START) + S(AMS_START), \ + S(CHUNK_NOT_SUPP) #define FOREACH_AMS(S) \ S(NONE_AMS), \ @@ -433,6 +434,7 @@ struct tcpm_port { /* Collision Avoidance and Atomic Message Sequence */ enum tcpm_state upcoming_state; enum tcpm_ams ams; + enum tcpm_ams next_ams; bool in_ams; #ifdef CONFIG_DEBUG_FS @@ -1213,7 +1215,8 @@ static int tcpm_ams_start(struct tcpm_port *port, enum tcpm_ams ams) tcpm_log(port, "AMS %s start", tcpm_ams_str[ams]); - if (!tcpm_ams_interruptible(port) && ams != HARD_RESET) { + if (!tcpm_ams_interruptible(port) && + !(ams == HARD_RESET || ams == SOFT_RESET_AMS)) { port->upcoming_state = INVALID_STATE; tcpm_log(port, "AMS %s not interruptible, aborting", tcpm_ams_str[port->ams]); @@ -1231,11 +1234,10 @@ static int tcpm_ams_start(struct tcpm_port *port, enum tcpm_ams ams) tcpm_set_state(port, HARD_RESET_START, 0); return ret; } else if (ams == SOFT_RESET_AMS) { - if (!port->explicit_contract) { - port->upcoming_state = INVALID_STATE; + if (!port->explicit_contract) tcpm_set_cc(port, tcpm_rp_cc(port)); - return ret; - } + tcpm_set_state(port, SOFT_RESET_SEND, 0); + return ret; } else if (tcpm_vdm_ams(port)) { /* tSinkTx is enforced in vdm_run_state_machine */ if (port->negotiated_rev >= PD_REV30) @@ -1452,6 +1454,9 @@ static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev, case CMDT_INIT: switch (cmd) { case CMD_DISCOVER_IDENT: + if (PD_VDO_VID(p[0]) != USB_SID_PD) + break; + /* 6.4.4.3.1: Only respond as UFP (device) */ if (port->data_role == TYPEC_DEVICE && port->nr_snk_vdo) { @@ -1537,22 +1542,37 @@ static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev, return 0; } break; + case VDO_CMD_VENDOR(0) ... VDO_CMD_VENDOR(15): + break; default: + /* Unrecognized SVDM */ + response[0] = p[0] | VDO_CMDT(CMDT_RSP_NAK); + rlen = 1; break; } break; case CMDT_RSP_NAK: tcpm_ams_finish(port); switch (cmd) { + case CMD_DISCOVER_IDENT: + case CMD_DISCOVER_SVID: + case CMD_DISCOVER_MODES: + case VDO_CMD_VENDOR(0) ... VDO_CMD_VENDOR(15): + break; case CMD_ENTER_MODE: /* Back to USB Operation */ *adev_action = ADEV_NOTIFY_USB_AND_QUEUE_VDM; return 0; default: + /* Unrecognized SVDM */ + response[0] = p[0] | VDO_CMDT(CMDT_RSP_NAK); + rlen = 1; break; } break; default: + response[0] = p[0] | VDO_CMDT(CMDT_RSP_NAK); + rlen = 1; break; } @@ -1588,8 +1608,12 @@ static void tcpm_handle_vdm_request(struct tcpm_port *port, port->vdm_state = VDM_STATE_DONE; } - if (PD_VDO_SVDM(p[0])) + if (PD_VDO_SVDM(p[0])) { rlen = tcpm_pd_svdm(port, adev, p, cnt, response, &adev_action); + } else { + if (port->negotiated_rev >= PD_REV30) + tcpm_queue_message(port, PD_MSG_CTRL_NOT_SUPP); + } /* * We are done with any state stored in the port struct now, except @@ -2039,6 +2063,71 @@ static int tcpm_set_auto_vbus_discharge_threshold(struct tcpm_port *port, return ret; } +static void tcpm_pd_handle_state(struct tcpm_port *port, + enum tcpm_state state, + enum tcpm_ams ams, + unsigned int delay_ms) +{ + switch (port->state) { + case SRC_READY: + case SNK_READY: + port->ams = ams; + tcpm_set_state(port, state, delay_ms); + break; + /* 8.3.3.4.1.1 and 6.8.1 power transitioning */ + case SNK_TRANSITION_SINK: + case SNK_TRANSITION_SINK_VBUS: + case SRC_TRANSITION_SUPPLY: + tcpm_set_state(port, HARD_RESET_SEND, 0); + break; + default: + if (!tcpm_ams_interruptible(port)) { + tcpm_set_state(port, port->pwr_role == TYPEC_SOURCE ? + SRC_SOFT_RESET_WAIT_SNK_TX : + SNK_SOFT_RESET, + 0); + } else { + /* process the Message 6.8.1 */ + port->upcoming_state = state; + port->next_ams = ams; + tcpm_set_state(port, ready_state(port), delay_ms); + } + break; + } +} + +static void tcpm_pd_handle_msg(struct tcpm_port *port, + enum pd_msg_request message, + enum tcpm_ams ams) +{ + switch (port->state) { + case SRC_READY: + case SNK_READY: + port->ams = ams; + tcpm_queue_message(port, message); + break; + /* PD 3.0 Spec 8.3.3.4.1.1 and 6.8.1 */ + case SNK_TRANSITION_SINK: + case SNK_TRANSITION_SINK_VBUS: + case SRC_TRANSITION_SUPPLY: + tcpm_set_state(port, HARD_RESET_SEND, 0); + break; + default: + if (!tcpm_ams_interruptible(port)) { + tcpm_set_state(port, port->pwr_role == TYPEC_SOURCE ? + SRC_SOFT_RESET_WAIT_SNK_TX : + SNK_SOFT_RESET, + 0); + } else { + port->next_ams = ams; + tcpm_set_state(port, ready_state(port), 0); + /* 6.8.1 process the Message */ + tcpm_queue_message(port, message); + } + break; + } +} + static void tcpm_pd_data_request(struct tcpm_port *port, const struct pd_message *msg) { @@ -2052,9 +2141,6 @@ static void tcpm_pd_data_request(struct tcpm_port *port, switch (type) { case PD_DATA_SOURCE_CAP: - if (port->pwr_role != TYPEC_SINK) - break; - for (i = 0; i < cnt; i++) port->source_caps[i] = le32_to_cpu(msg->payload[i]); @@ -2070,12 +2156,26 @@ static void tcpm_pd_data_request(struct tcpm_port *port, * to comply with 6.2.1.1.5 of the USB PD 3.0 spec. We don't * support Rev 1.0 so just do nothing in that scenario. */ - if (rev == PD_REV10) + if (rev == PD_REV10) { + if (port->ams == GET_SOURCE_CAPABILITIES) + tcpm_ams_finish(port); break; + } if (rev < PD_MAX_REV) port->negotiated_rev = rev; + if (port->pwr_role == TYPEC_SOURCE) { + if (port->ams == GET_SOURCE_CAPABILITIES) + tcpm_pd_handle_state(port, SRC_READY, NONE_AMS, 0); + /* Unexpected Source Capabilities */ + else + tcpm_pd_handle_msg(port, + port->negotiated_rev < PD_REV30 ? + PD_MSG_CTRL_REJECT : + PD_MSG_CTRL_NOT_SUPP, + NONE_AMS); + } else if (port->state == SNK_WAIT_CAPABILITIES) { /* * This message may be received even if VBUS is not * present. This is quite unexpected; see USB PD @@ -2089,30 +2189,48 @@ static void tcpm_pd_data_request(struct tcpm_port *port, * but be prepared to keep waiting for VBUS after it was * handled. */ - tcpm_set_state(port, SNK_NEGOTIATE_CAPABILITIES, 0); + port->ams = POWER_NEGOTIATION; + tcpm_set_state(port, SNK_NEGOTIATE_CAPABILITIES, 0); + } else { + if (port->ams == GET_SOURCE_CAPABILITIES) + tcpm_ams_finish(port); + tcpm_pd_handle_state(port, SNK_NEGOTIATE_CAPABILITIES, + POWER_NEGOTIATION, 0); + } break; case PD_DATA_REQUEST: - if (port->pwr_role != TYPEC_SOURCE || - cnt != 1) { - tcpm_queue_message(port, PD_MSG_CTRL_REJECT); - break; - } - /* * Adjust revision in subsequent message headers, as required, * to comply with 6.2.1.1.5 of the USB PD 3.0 spec. We don't * support Rev 1.0 so just reject in that scenario. */ if (rev == PD_REV10) { - tcpm_queue_message(port, PD_MSG_CTRL_REJECT); + tcpm_pd_handle_msg(port, + port->negotiated_rev < PD_REV30 ? + PD_MSG_CTRL_REJECT : + PD_MSG_CTRL_NOT_SUPP, + NONE_AMS); break; } if (rev < PD_MAX_REV) port->negotiated_rev = rev; + if (port->pwr_role != TYPEC_SOURCE || cnt != 1) { + tcpm_pd_handle_msg(port, + port->negotiated_rev < PD_REV30 ? + PD_MSG_CTRL_REJECT : + PD_MSG_CTRL_NOT_SUPP, + NONE_AMS); + break; + } + port->sink_request = le32_to_cpu(msg->payload[0]); - tcpm_set_state(port, SRC_NEGOTIATE_CAPABILITIES, 0); + if (port->state == SRC_SEND_CAPABILITIES) + tcpm_set_state(port, SRC_NEGOTIATE_CAPABILITIES, 0); + else + tcpm_pd_handle_state(port, SRC_NEGOTIATE_CAPABILITIES, + POWER_NEGOTIATION, 0); break; case PD_DATA_SINK_CAP: /* We don't do anything with this at the moment... */ @@ -2133,16 +2251,22 @@ static void tcpm_pd_data_request(struct tcpm_port *port, port->nr_sink_caps = cnt; port->sink_cap_done = true; - tcpm_set_state(port, SNK_READY, 0); + if (port->ams == GET_SINK_CAPABILITIES) + tcpm_pd_handle_state(port, ready_state(port), NONE_AMS, 0); + /* Unexpected Sink Capabilities */ + else + tcpm_pd_handle_msg(port, + port->negotiated_rev < PD_REV30 ? + PD_MSG_CTRL_REJECT : + PD_MSG_CTRL_NOT_SUPP, + NONE_AMS); break; case PD_DATA_VENDOR_DEF: tcpm_handle_vdm_request(port, msg->payload, cnt); break; case PD_DATA_BIST: - if (port->state == SRC_READY || port->state == SNK_READY) { - port->bist_request = le32_to_cpu(msg->payload[0]); - tcpm_set_state(port, BIST_RX, 0); - } + port->bist_request = le32_to_cpu(msg->payload[0]); + tcpm_pd_handle_state(port, BIST_RX, BIST, 0); break; case PD_DATA_ALERT: tcpm_handle_alert(port, msg->payload, cnt); @@ -2150,10 +2274,17 @@ static void tcpm_pd_data_request(struct tcpm_port *port, case PD_DATA_BATT_STATUS: case PD_DATA_GET_COUNTRY_INFO: /* Currently unsupported */ - tcpm_queue_message(port, PD_MSG_CTRL_NOT_SUPP); + tcpm_pd_handle_msg(port, port->negotiated_rev < PD_REV30 ? + PD_MSG_CTRL_REJECT : + PD_MSG_CTRL_NOT_SUPP, + NONE_AMS); break; default: - tcpm_log(port, "Unhandled data message type %#x", type); + tcpm_pd_handle_msg(port, port->negotiated_rev < PD_REV30 ? + PD_MSG_CTRL_REJECT : + PD_MSG_CTRL_NOT_SUPP, + NONE_AMS); + tcpm_log(port, "Unrecognized data message type %#x", type); break; } } @@ -2178,26 +2309,10 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port, case PD_CTRL_PING: break; case PD_CTRL_GET_SOURCE_CAP: - switch (port->state) { - case SRC_READY: - case SNK_READY: - tcpm_queue_message(port, PD_MSG_DATA_SOURCE_CAP); - break; - default: - tcpm_queue_message(port, PD_MSG_CTRL_REJECT); - break; - } + tcpm_pd_handle_msg(port, PD_MSG_DATA_SOURCE_CAP, GET_SOURCE_CAPABILITIES); break; case PD_CTRL_GET_SINK_CAP: - switch (port->state) { - case SRC_READY: - case SNK_READY: - tcpm_queue_message(port, PD_MSG_DATA_SINK_CAP); - break; - default: - tcpm_queue_message(port, PD_MSG_CTRL_REJECT); - break; - } + tcpm_pd_handle_msg(port, PD_MSG_DATA_SINK_CAP, GET_SINK_CAPABILITIES); break; case PD_CTRL_GOTO_MIN: break; @@ -2236,6 +2351,11 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port, tcpm_set_state(port, FR_SWAP_SNK_SRC_NEW_SINK_READY, 0); break; default: + tcpm_pd_handle_state(port, + port->pwr_role == TYPEC_SOURCE ? + SRC_SOFT_RESET_WAIT_SNK_TX : + SNK_SOFT_RESET, + NONE_AMS, 0); break; } break; @@ -2282,6 +2402,11 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port, tcpm_set_state(port, ready_state(port), 0); break; default: + tcpm_pd_handle_state(port, + port->pwr_role == TYPEC_SOURCE ? + SRC_SOFT_RESET_WAIT_SNK_TX : + SNK_SOFT_RESET, + NONE_AMS, 0); break; } break; @@ -2298,8 +2423,6 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port, tcpm_set_state(port, SNK_TRANSITION_SINK, 0); break; case SOFT_RESET_SEND: - port->message_id = 0; - port->rx_msgid = -1; if (port->ams == SOFT_RESET_AMS) tcpm_ams_finish(port); if (port->pwr_role == TYPEC_SOURCE) { @@ -2322,57 +2445,45 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port, tcpm_set_state(port, FR_SWAP_SNK_SRC_TRANSITION_TO_OFF, 0); break; default: + tcpm_pd_handle_state(port, + port->pwr_role == TYPEC_SOURCE ? + SRC_SOFT_RESET_WAIT_SNK_TX : + SNK_SOFT_RESET, + NONE_AMS, 0); break; } break; case PD_CTRL_SOFT_RESET: + port->ams = SOFT_RESET_AMS; tcpm_set_state(port, SOFT_RESET, 0); break; case PD_CTRL_DR_SWAP: - if (port->typec_caps.data != TYPEC_PORT_DRD) { - tcpm_queue_message(port, PD_MSG_CTRL_REJECT); - break; - } /* * XXX * 6.3.9: If an alternate mode is active, a request to swap * alternate modes shall trigger a port reset. */ - switch (port->state) { - case SRC_READY: - case SNK_READY: - tcpm_set_state(port, DR_SWAP_ACCEPT, 0); - break; - default: - tcpm_queue_message(port, PD_MSG_CTRL_WAIT); - break; - } + if (port->typec_caps.data != TYPEC_PORT_DRD) + tcpm_pd_handle_msg(port, + port->negotiated_rev < PD_REV30 ? + PD_MSG_CTRL_REJECT : + PD_MSG_CTRL_NOT_SUPP, + NONE_AMS); + else + tcpm_pd_handle_state(port, DR_SWAP_ACCEPT, DATA_ROLE_SWAP, 0); break; case PD_CTRL_PR_SWAP: - if (port->port_type != TYPEC_PORT_DRP) { - tcpm_queue_message(port, PD_MSG_CTRL_REJECT); - break; - } - switch (port->state) { - case SRC_READY: - case SNK_READY: - tcpm_set_state(port, PR_SWAP_ACCEPT, 0); - break; - default: - tcpm_queue_message(port, PD_MSG_CTRL_WAIT); - break; - } + if (port->port_type != TYPEC_PORT_DRP) + tcpm_pd_handle_msg(port, + port->negotiated_rev < PD_REV30 ? + PD_MSG_CTRL_REJECT : + PD_MSG_CTRL_NOT_SUPP, + NONE_AMS); + else + tcpm_pd_handle_state(port, PR_SWAP_ACCEPT, POWER_ROLE_SWAP, 0); break; case PD_CTRL_VCONN_SWAP: - switch (port->state) { - case SRC_READY: - case SNK_READY: - tcpm_set_state(port, VCONN_SWAP_ACCEPT, 0); - break; - default: - tcpm_queue_message(port, PD_MSG_CTRL_WAIT); - break; - } + tcpm_pd_handle_state(port, VCONN_SWAP_ACCEPT, VCONN_SWAP, 0); break; case PD_CTRL_GET_SOURCE_CAP_EXT: case PD_CTRL_GET_STATUS: @@ -2380,10 +2491,19 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port, case PD_CTRL_GET_PPS_STATUS: case PD_CTRL_GET_COUNTRY_CODES: /* Currently not supported */ - tcpm_queue_message(port, PD_MSG_CTRL_NOT_SUPP); + tcpm_pd_handle_msg(port, + port->negotiated_rev < PD_REV30 ? + PD_MSG_CTRL_REJECT : + PD_MSG_CTRL_NOT_SUPP, + NONE_AMS); break; default: - tcpm_log(port, "Unhandled ctrl message type %#x", type); + tcpm_pd_handle_msg(port, + port->negotiated_rev < PD_REV30 ? + PD_MSG_CTRL_REJECT : + PD_MSG_CTRL_NOT_SUPP, + NONE_AMS); + tcpm_log(port, "Unrecognized ctrl message type %#x", type); break; } } @@ -2395,11 +2515,13 @@ static void tcpm_pd_ext_msg_request(struct tcpm_port *port, unsigned int data_size = pd_ext_header_data_size_le(msg->ext_msg.header); if (!(msg->ext_msg.header & PD_EXT_HDR_CHUNKED)) { + tcpm_pd_handle_msg(port, PD_MSG_CTRL_NOT_SUPP, NONE_AMS); tcpm_log(port, "Unchunked extended messages unsupported"); return; } if (data_size > PD_EXT_MAX_CHUNK_DATA) { + tcpm_pd_handle_state(port, CHUNK_NOT_SUPP, NONE_AMS, PD_T_CHUNK_NOT_SUPP); tcpm_log(port, "Chunk handling not yet supported"); return; } @@ -2412,16 +2534,18 @@ static void tcpm_pd_ext_msg_request(struct tcpm_port *port, */ if (msg->ext_msg.data[USB_PD_EXT_SDB_EVENT_FLAGS] & USB_PD_EXT_SDB_PPS_EVENTS) - tcpm_set_state(port, GET_PPS_STATUS_SEND, 0); + tcpm_pd_handle_state(port, GET_PPS_STATUS_SEND, + GETTING_SOURCE_SINK_STATUS, 0); + else - tcpm_set_state(port, ready_state(port), 0); + tcpm_pd_handle_state(port, ready_state(port), NONE_AMS, 0); break; case PD_EXT_PPS_STATUS: /* * For now the PPS status message is used to clear events * and nothing more. */ - tcpm_set_state(port, ready_state(port), 0); + tcpm_pd_handle_state(port, ready_state(port), NONE_AMS, 0); break; case PD_EXT_SOURCE_CAP_EXT: case PD_EXT_GET_BATT_CAP: @@ -2435,10 +2559,11 @@ static void tcpm_pd_ext_msg_request(struct tcpm_port *port, case PD_EXT_FW_UPDATE_RESPONSE: case PD_EXT_COUNTRY_INFO: case PD_EXT_COUNTRY_CODES: - tcpm_queue_message(port, PD_MSG_CTRL_NOT_SUPP); + tcpm_pd_handle_msg(port, PD_MSG_CTRL_NOT_SUPP, NONE_AMS); break; default: - tcpm_log(port, "Unhandled extended message type %#x", type); + tcpm_pd_handle_msg(port, PD_MSG_CTRL_NOT_SUPP, NONE_AMS); + tcpm_log(port, "Unrecognized extended message type %#x", type); break; } } @@ -2551,7 +2676,12 @@ static bool tcpm_send_queued_message(struct tcpm_port *port) tcpm_pd_send_control(port, PD_CTRL_NOT_SUPP); break; case PD_MSG_DATA_SINK_CAP: - tcpm_pd_send_sink_caps(port); + ret = tcpm_pd_send_sink_caps(port); + if (ret < 0) { + tcpm_log(port, "Unable to send snk caps, ret=%d", ret); + tcpm_set_state(port, SNK_SOFT_RESET, 0); + } + tcpm_ams_finish(port); break; case PD_MSG_DATA_SOURCE_CAP: ret = tcpm_pd_send_source_caps(port); @@ -2561,8 +2691,11 @@ static bool tcpm_send_queued_message(struct tcpm_port *port) ret); tcpm_set_state(port, SOFT_RESET_SEND, 0); } else if (port->pwr_role == TYPEC_SOURCE) { + tcpm_ams_finish(port); tcpm_set_state(port, HARD_RESET_SEND, PD_T_SENDER_RESPONSE); + } else { + tcpm_ams_finish(port); } break; default: @@ -3581,6 +3714,11 @@ static void run_state_machine(struct tcpm_port *port) if (port->ams != NONE_AMS) tcpm_ams_finish(port); + if (port->next_ams != NONE_AMS) { + port->ams = port->next_ams; + port->next_ams = NONE_AMS; + } + /* * If previous AMS is interrupted, switch to the upcoming * state. @@ -3821,6 +3959,11 @@ static void run_state_machine(struct tcpm_port *port) if (port->ams != NONE_AMS) tcpm_ams_finish(port); + if (port->next_ams != NONE_AMS) { + port->ams = port->next_ams; + port->next_ams = NONE_AMS; + } + /* * If previous AMS is interrupted, switch to the upcoming * state. @@ -3968,6 +4111,7 @@ static void run_state_machine(struct tcpm_port *port) port->message_id = 0; port->rx_msgid = -1; tcpm_pd_send_control(port, PD_CTRL_ACCEPT); + tcpm_ams_finish(port); if (port->pwr_role == TYPEC_SOURCE) { port->upcoming_state = SRC_SEND_CAPABILITIES; tcpm_ams_start(port, POWER_NEGOTIATION); @@ -4004,6 +4148,7 @@ static void run_state_machine(struct tcpm_port *port) break; case DR_SWAP_SEND_TIMEOUT: tcpm_swap_complete(port, -ETIMEDOUT); + tcpm_ams_finish(port); tcpm_set_state(port, ready_state(port), 0); break; case DR_SWAP_CHANGE_DR: @@ -4016,6 +4161,7 @@ static void run_state_machine(struct tcpm_port *port) TYPEC_HOST); port->send_discover = true; } + tcpm_ams_finish(port); tcpm_set_state(port, ready_state(port), 0); break; @@ -4143,6 +4289,7 @@ static void run_state_machine(struct tcpm_port *port) case VCONN_SWAP_ACCEPT: tcpm_pd_send_control(port, PD_CTRL_ACCEPT); + tcpm_ams_finish(port); tcpm_set_state(port, VCONN_SWAP_START, 0); break; case VCONN_SWAP_SEND: @@ -4260,6 +4407,12 @@ static void run_state_machine(struct tcpm_port *port) port->upcoming_state = INVALID_STATE; tcpm_set_state(port, upcoming_state, 0); break; + + /* Chunk state */ + case CHUNK_NOT_SUPP: + tcpm_pd_send_control(port, PD_CTRL_NOT_SUPP); + tcpm_set_state(port, port->pwr_role == TYPEC_SOURCE ? SRC_READY : SNK_READY, 0); + break; default: WARN(1, "Unexpected port state %d\n", port->state); break; @@ -4689,6 +4842,8 @@ static void _tcpm_pd_hard_reset(struct tcpm_port *port) if (port->ams != NONE_AMS) port->ams = NONE_AMS; + if (port->hard_reset_count < PD_N_HARD_RESET_COUNT) + port->ams = HARD_RESET; /* * If we keep receiving hard reset requests, executing the hard reset * must have failed. Revert to error recovery if that happens. diff --git a/include/linux/usb/pd.h b/include/linux/usb/pd.h index 79599b90ba55..272454f9cd67 100644 --- a/include/linux/usb/pd.h +++ b/include/linux/usb/pd.h @@ -480,6 +480,7 @@ static inline unsigned int rdo_max_power(u32 rdo) #define PD_T_SWAP_SRC_START 20 /* Minimum of 20ms */ #define PD_T_BIST_CONT_MODE 50 /* 30 - 60 ms */ #define PD_T_SINK_TX 16 /* 16 - 20 ms */ +#define PD_T_CHUNK_NOT_SUPP 42 /* 40 - 50 ms */ #define PD_T_DRP_TRY 100 /* 75 - 150 ms */ #define PD_T_DRP_TRYWAIT 600 /* 400 - 800 ms */ -- cgit v1.2.3 From 742d33729a0df11c9d8d4625dbf21dd20cdefd44 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 20 Jan 2021 14:34:23 +0000 Subject: mm: Move immutable fields of 'struct vm_fault' into anonymous struct 'struct vm_fault' contains both information about the fault being serviced alongside mutable fields contributing to the state of the fault-handling logic. Unfortunately, the distinction between the two is not clear-cut, and a number of callers end up manipulating the structure temporarily before restoring it when returning. Try to clean this up by moving the immutable fault information into an anonymous struct, which will later be marked as 'const'. Ideally, the 'flags' field would be part of the new structure too, but it seems as though the ->page_mkwrite() path is not ready for this yet. Cc: Kirill A. Shutemov Suggested-by: Linus Torvalds Link: https://lore.kernel.org/r/CAHk-=whYs9XsO88iqJzN6NC=D-dp2m0oYXuOoZ=eWnvv=5OA+w@mail.gmail.com Signed-off-by: Will Deacon --- include/linux/mm.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 251a2339befb..b4a5cb9bff7d 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -517,11 +517,14 @@ static inline bool fault_flag_allow_retry_first(unsigned int flags) * pgoff should be used in favour of virtual_address, if possible. */ struct vm_fault { - struct vm_area_struct *vma; /* Target VMA */ - unsigned int flags; /* FAULT_FLAG_xxx flags */ - gfp_t gfp_mask; /* gfp mask to be used for allocations */ - pgoff_t pgoff; /* Logical page offset based on vma */ - unsigned long address; /* Faulting virtual address */ + struct { + struct vm_area_struct *vma; /* Target VMA */ + gfp_t gfp_mask; /* gfp mask to be used for allocations */ + pgoff_t pgoff; /* Logical page offset based on vma */ + unsigned long address; /* Faulting virtual address */ + }; + unsigned int flags; /* FAULT_FLAG_xxx flags + * XXX: should really be 'const' */ pmd_t *pmd; /* Pointer to pmd entry matching * the 'address' */ pud_t *pud; /* Pointer to pud entry matching -- cgit v1.2.3 From 9d3af4b448a119ac81378d3bc775f1c4a2a7ff36 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 14 Jan 2021 15:24:19 +0000 Subject: mm: Pass 'address' to map to do_set_pte() and drop FAULT_FLAG_PREFAULT Rather than modifying the 'address' field of the 'struct vm_fault' passed to do_set_pte(), leave that to identify the real faulting address and pass in the virtual address to be mapped by the new pte as a separate argument. This makes FAULT_FLAG_PREFAULT redundant, as a prefault entry can be identified simply by comparing the new address parameter with the faulting address, so remove the redundant flag at the same time. Cc: Kirill A. Shutemov Cc: Linus Torvalds Signed-off-by: Will Deacon --- include/linux/mm.h | 7 ++----- mm/filemap.c | 21 +++++++-------------- mm/memory.c | 10 +++++----- 3 files changed, 14 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index b4a5cb9bff7d..e0f056753bef 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -434,7 +434,6 @@ extern pgprot_t protection_map[16]; * @FAULT_FLAG_REMOTE: The fault is not for current task/mm. * @FAULT_FLAG_INSTRUCTION: The fault was during an instruction fetch. * @FAULT_FLAG_INTERRUPTIBLE: The fault can be interrupted by non-fatal signals. - * @FAULT_FLAG_PREFAULT: Fault was a prefault. * * About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify * whether we would allow page faults to retry by specifying these two @@ -465,7 +464,6 @@ extern pgprot_t protection_map[16]; #define FAULT_FLAG_REMOTE 0x80 #define FAULT_FLAG_INSTRUCTION 0x100 #define FAULT_FLAG_INTERRUPTIBLE 0x200 -#define FAULT_FLAG_PREFAULT 0x400 /* * The default fault flags that should be used by most of the @@ -503,8 +501,7 @@ static inline bool fault_flag_allow_retry_first(unsigned int flags) { FAULT_FLAG_USER, "USER" }, \ { FAULT_FLAG_REMOTE, "REMOTE" }, \ { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \ - { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \ - { FAULT_FLAG_PREFAULT, "PREFAULT" } + { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" } /* * vm_fault is filled by the pagefault handler and passed to the vma's @@ -995,7 +992,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) } vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page); -void do_set_pte(struct vm_fault *vmf, struct page *page); +void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr); vm_fault_t finish_fault(struct vm_fault *vmf); vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf); diff --git a/mm/filemap.c b/mm/filemap.c index a6dc97906c8e..fb7a8d9b5603 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3018,8 +3018,7 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, struct file *file = vma->vm_file; struct address_space *mapping = file->f_mapping; pgoff_t last_pgoff = start_pgoff; - unsigned long address = vmf->address; - unsigned long flags = vmf->flags; + unsigned long addr; XA_STATE(xas, &mapping->i_pages, start_pgoff); struct page *head, *page; unsigned int mmap_miss = READ_ONCE(file->f_ra.mmap_miss); @@ -3035,8 +3034,8 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, goto out; } - vmf->address = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); - vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); + addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); + vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl); do { page = find_subpage(head, xas.xa_index); if (PageHWPoison(page)) @@ -3045,7 +3044,7 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, if (mmap_miss > 0) mmap_miss--; - vmf->address += (xas.xa_index - last_pgoff) << PAGE_SHIFT; + addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT; vmf->pte += xas.xa_index - last_pgoff; last_pgoff = xas.xa_index; @@ -3053,16 +3052,12 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, goto unlock; /* We're about to handle the fault */ - if (vmf->address == address) { - vmf->flags &= ~FAULT_FLAG_PREFAULT; + if (vmf->address == addr) ret = VM_FAULT_NOPAGE; - } else { - vmf->flags |= FAULT_FLAG_PREFAULT; - } - do_set_pte(vmf, page); + do_set_pte(vmf, page, addr); /* no need to invalidate: a not-present page won't be cached */ - update_mmu_cache(vma, vmf->address, vmf->pte); + update_mmu_cache(vma, addr, vmf->pte); unlock_page(head); continue; unlock: @@ -3072,8 +3067,6 @@ unlock: pte_unmap_unlock(vmf->pte, vmf->ptl); out: rcu_read_unlock(); - vmf->flags = flags; - vmf->address = address; WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss); return ret; } diff --git a/mm/memory.c b/mm/memory.c index f0e7c589ca9d..7b1307873325 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3733,11 +3733,11 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page) } #endif -void do_set_pte(struct vm_fault *vmf, struct page *page) +void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr) { struct vm_area_struct *vma = vmf->vma; bool write = vmf->flags & FAULT_FLAG_WRITE; - bool prefault = vmf->flags & FAULT_FLAG_PREFAULT; + bool prefault = vmf->address != addr; pte_t entry; flush_icache_page(vma, page); @@ -3753,13 +3753,13 @@ void do_set_pte(struct vm_fault *vmf, struct page *page) /* copy-on-write page */ if (write && !(vma->vm_flags & VM_SHARED)) { inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); - page_add_new_anon_rmap(page, vma, vmf->address, false); + page_add_new_anon_rmap(page, vma, addr, false); lru_cache_add_inactive_or_unevictable(page, vma); } else { inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page)); page_add_file_rmap(page, false); } - set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry); + set_pte_at(vma->vm_mm, addr, vmf->pte, entry); } /** @@ -3819,7 +3819,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf) ret = 0; /* Re-check under ptl */ if (likely(pte_none(*vmf->pte))) - do_set_pte(vmf, page); + do_set_pte(vmf, page, vmf->address); else ret = VM_FAULT_NOPAGE; -- cgit v1.2.3 From 5857c9209ce58f8e262889539ccdf63e73ad7a93 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 14 Jan 2021 15:44:09 +0000 Subject: mm: Mark anonymous struct field of 'struct vm_fault' as 'const' The fields of this struct are only ever read after being initialised, so mark it 'const' before somebody tries to modify it again. GCC will then complain (with an error) about modification of these fields after they have been initialised, although LLVM currently allows them without even a warning: https://bugs.llvm.org/show_bug.cgi?id=48755 Hopefully, future versions of LLVM will emit a warning. Cc: Kirill A. Shutemov Cc: Linus Torvalds Signed-off-by: Will Deacon --- include/linux/mm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index e0f056753bef..7ff3d9817d38 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -514,7 +514,7 @@ static inline bool fault_flag_allow_retry_first(unsigned int flags) * pgoff should be used in favour of virtual_address, if possible. */ struct vm_fault { - struct { + const struct { struct vm_area_struct *vma; /* Target VMA */ gfp_t gfp_mask; /* gfp mask to be used for allocations */ pgoff_t pgoff; /* Logical page offset based on vma */ -- cgit v1.2.3 From 239319670e2a7c405eeb4b3e7721cf8bf8eef840 Mon Sep 17 00:00:00 2001 From: Ye Xiang Date: Tue, 15 Dec 2020 13:44:42 +0800 Subject: HID: hid-sensor-custom: Add custom sensor iio support Currently custom sensors properties are not decoded and it is up to user space to interpret. Some manufacturers already standardized the meaning of some custom sensors. They can be presented as a proper IIO sensor. We can identify these sensors based on manufacturer and serial number property in the report. This change is identifying hinge sensor when the manufacturer is "INTEL". This creates a platform device so that a sensor driver can be loaded to process these sensors. Signed-off-by: Ye Xiang Acked-by: Jiri Kosina Link: https://lore.kernel.org/r/20201215054444.9324-2-xiang.ye@intel.com Signed-off-by: Jonathan Cameron --- drivers/hid/hid-sensor-custom.c | 143 ++++++++++++++++++++++++++++++++++++++++ include/linux/hid-sensor-ids.h | 14 ++++ 2 files changed, 157 insertions(+) (limited to 'include/linux') diff --git a/drivers/hid/hid-sensor-custom.c b/drivers/hid/hid-sensor-custom.c index 4d25577a8573..2628bc53ed80 100644 --- a/drivers/hid/hid-sensor-custom.c +++ b/drivers/hid/hid-sensor-custom.c @@ -4,6 +4,7 @@ * Copyright (c) 2015, Intel Corporation. */ +#include #include #include #include @@ -21,6 +22,7 @@ #define HID_CUSTOM_TOTAL_ATTRS (HID_CUSTOM_MAX_CORE_ATTRS + 1) #define HID_CUSTOM_FIFO_SIZE 4096 #define HID_CUSTOM_MAX_FEATURE_BYTES 64 +#define HID_SENSOR_USAGE_LENGTH (4 + 1) struct hid_sensor_custom_field { int report_id; @@ -50,6 +52,7 @@ struct hid_sensor_custom { struct kfifo data_fifo; unsigned long misc_opened; wait_queue_head_t wait; + struct platform_device *custom_pdev; }; /* Header for each sample to user space via dev interface */ @@ -746,11 +749,130 @@ static void hid_sensor_custom_dev_if_remove(struct hid_sensor_custom } +/* luid defined in FW (e.g. ISH). Maybe used to identify sensor. */ +static const char *const known_sensor_luid[] = { "020B000000000000" }; + +static int get_luid_table_index(unsigned char *usage_str) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(known_sensor_luid); i++) { + if (!strncmp(usage_str, known_sensor_luid[i], + strlen(known_sensor_luid[i]))) + return i; + } + + return -ENODEV; +} + +static int get_known_custom_sensor_index(struct hid_sensor_hub_device *hsdev) +{ + struct hid_sensor_hub_attribute_info sensor_manufacturer = { 0 }; + struct hid_sensor_hub_attribute_info sensor_luid_info = { 0 }; + int report_size; + int ret; + static u16 w_buf[HID_CUSTOM_MAX_FEATURE_BYTES]; + static char buf[HID_CUSTOM_MAX_FEATURE_BYTES]; + int i; + + memset(w_buf, 0, sizeof(w_buf)); + memset(buf, 0, sizeof(buf)); + + /* get manufacturer info */ + ret = sensor_hub_input_get_attribute_info(hsdev, + HID_FEATURE_REPORT, hsdev->usage, + HID_USAGE_SENSOR_PROP_MANUFACTURER, &sensor_manufacturer); + if (ret < 0) + return ret; + + report_size = + sensor_hub_get_feature(hsdev, sensor_manufacturer.report_id, + sensor_manufacturer.index, sizeof(w_buf), + w_buf); + if (report_size <= 0) { + hid_err(hsdev->hdev, + "Failed to get sensor manufacturer info %d\n", + report_size); + return -ENODEV; + } + + /* convert from wide char to char */ + for (i = 0; i < ARRAY_SIZE(buf) - 1 && w_buf[i]; i++) + buf[i] = (char)w_buf[i]; + + /* ensure it's ISH sensor */ + if (strncmp(buf, "INTEL", strlen("INTEL"))) + return -ENODEV; + + memset(w_buf, 0, sizeof(w_buf)); + memset(buf, 0, sizeof(buf)); + + /* get real usage id */ + ret = sensor_hub_input_get_attribute_info(hsdev, + HID_FEATURE_REPORT, hsdev->usage, + HID_USAGE_SENSOR_PROP_SERIAL_NUM, &sensor_luid_info); + if (ret < 0) + return ret; + + report_size = sensor_hub_get_feature(hsdev, sensor_luid_info.report_id, + sensor_luid_info.index, sizeof(w_buf), + w_buf); + if (report_size <= 0) { + hid_err(hsdev->hdev, "Failed to get real usage info %d\n", + report_size); + return -ENODEV; + } + + /* convert from wide char to char */ + for (i = 0; i < ARRAY_SIZE(buf) - 1 && w_buf[i]; i++) + buf[i] = (char)w_buf[i]; + + if (strlen(buf) != strlen(known_sensor_luid[0]) + 5) { + hid_err(hsdev->hdev, + "%s luid length not match %zu != (%zu + 5)\n", __func__, + strlen(buf), strlen(known_sensor_luid[0])); + return -ENODEV; + } + + /* get table index with luid (not matching 'LUID: ' in luid) */ + return get_luid_table_index(&buf[5]); +} + +static struct platform_device * +hid_sensor_register_platform_device(struct platform_device *pdev, + struct hid_sensor_hub_device *hsdev, + int index) +{ + char real_usage[HID_SENSOR_USAGE_LENGTH] = { 0 }; + struct platform_device *custom_pdev; + const char *dev_name; + char *c; + + /* copy real usage id */ + memcpy(real_usage, known_sensor_luid[index], 4); + + /* usage id are all lowcase */ + for (c = real_usage; *c != '\0'; c++) + *c = tolower(*c); + + /* HID-SENSOR-INT-REAL_USAGE_ID */ + dev_name = kasprintf(GFP_KERNEL, "HID-SENSOR-INT-%s", real_usage); + if (!dev_name) + return ERR_PTR(-ENOMEM); + + custom_pdev = platform_device_register_data(pdev->dev.parent, dev_name, + PLATFORM_DEVID_NONE, hsdev, + sizeof(*hsdev)); + kfree(dev_name); + return custom_pdev; +} + static int hid_sensor_custom_probe(struct platform_device *pdev) { struct hid_sensor_custom *sensor_inst; struct hid_sensor_hub_device *hsdev = pdev->dev.platform_data; int ret; + int index; sensor_inst = devm_kzalloc(&pdev->dev, sizeof(*sensor_inst), GFP_KERNEL); @@ -764,6 +886,22 @@ static int hid_sensor_custom_probe(struct platform_device *pdev) sensor_inst->pdev = pdev; mutex_init(&sensor_inst->mutex); platform_set_drvdata(pdev, sensor_inst); + + index = get_known_custom_sensor_index(hsdev); + if (index >= 0 && index < ARRAY_SIZE(known_sensor_luid)) { + sensor_inst->custom_pdev = + hid_sensor_register_platform_device(pdev, hsdev, index); + + ret = PTR_ERR_OR_ZERO(sensor_inst->custom_pdev); + if (ret) { + dev_err(&pdev->dev, + "register_platform_device failed\n"); + return ret; + } + + return 0; + } + ret = sensor_hub_register_callback(hsdev, hsdev->usage, &sensor_inst->callbacks); if (ret < 0) { @@ -802,6 +940,11 @@ static int hid_sensor_custom_remove(struct platform_device *pdev) struct hid_sensor_custom *sensor_inst = platform_get_drvdata(pdev); struct hid_sensor_hub_device *hsdev = pdev->dev.platform_data; + if (sensor_inst->custom_pdev) { + platform_device_unregister(sensor_inst->custom_pdev); + return 0; + } + hid_sensor_custom_dev_if_remove(sensor_inst); hid_sensor_custom_remove_attributes(sensor_inst); sysfs_remove_group(&sensor_inst->pdev->dev.kobj, diff --git a/include/linux/hid-sensor-ids.h b/include/linux/hid-sensor-ids.h index d82a97e311d3..3bbdbccc5805 100644 --- a/include/linux/hid-sensor-ids.h +++ b/include/linux/hid-sensor-ids.h @@ -128,6 +128,10 @@ #define HID_USAGE_SENSOR_UNITS_DEGREES_PER_SECOND 0x15 /* Common selectors */ +#define HID_USAGE_SENSOR_PROP_DESC 0x200300 +#define HID_USAGE_SENSOR_PROP_FRIENDLY_NAME 0x200301 +#define HID_USAGE_SENSOR_PROP_SERIAL_NUM 0x200307 +#define HID_USAGE_SENSOR_PROP_MANUFACTURER 0x200305 #define HID_USAGE_SENSOR_PROP_REPORT_INTERVAL 0x20030E #define HID_USAGE_SENSOR_PROP_SENSITIVITY_ABS 0x20030F #define HID_USAGE_SENSOR_PROP_SENSITIVITY_RANGE_PCT 0x200310 @@ -158,4 +162,14 @@ #define HID_USAGE_SENSOR_PROP_REPORTING_STATE_NO_EVENTS_ENUM 0x200840 #define HID_USAGE_SENSOR_PROP_REPORTING_STATE_ALL_EVENTS_ENUM 0x200841 +/* Custom Sensor (2000e1) */ +#define HID_USAGE_SENSOR_HINGE 0x20020B +#define HID_USAGE_SENSOR_DATA_FIELD_LOCATION 0x200400 +#define HID_USAGE_SENSOR_DATA_FIELE_TIME_SINCE_SYS_BOOT 0x20052B +#define HID_USAGE_SENSOR_DATA_FIELD_CUSTOM_USAGE 0x200541 +#define HID_USAGE_SENSOR_DATA_FIELD_CUSTOM_VALUE_BASE 0x200543 +/* Custom Sensor data 28=>x>=0 */ +#define HID_USAGE_SENSOR_DATA_FIELD_CUSTOM_VALUE(x) \ + (HID_USAGE_SENSOR_DATA_FIELD_CUSTOM_VALUE_BASE + (x)) + #endif -- cgit v1.2.3 From 2f0df49c89acaa58571d509830bc481250699885 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Fri, 11 Dec 2020 16:37:54 -0500 Subject: jump_label: Do not profile branch annotations While running my branch profiler that checks for incorrect "likely" and "unlikely"s around the kernel, there's a large number of them that are incorrect due to being "static_branches". As static_branches are rather special, as they are likely or unlikely for other reasons than normal annotations are used for, there's no reason to have them be profiled. Expose the "unlikely_notrace" and "likely_notrace" so that the static_branch can use them, and have them be ignored by the branch profilers. Signed-off-by: Steven Rostedt (VMware) Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20201211163754.585174b9@gandalf.local.home --- include/linux/compiler.h | 2 ++ include/linux/jump_label.h | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/compiler.h b/include/linux/compiler.h index b8fe0c23cfff..df5b405e6305 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -76,6 +76,8 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, #else # define likely(x) __builtin_expect(!!(x), 1) # define unlikely(x) __builtin_expect(!!(x), 0) +# define likely_notrace(x) likely(x) +# define unlikely_notrace(x) unlikely(x) #endif /* Optimization barrier */ diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 32809624d422..d92691262f51 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -261,14 +261,14 @@ static __always_inline void jump_label_init(void) static __always_inline bool static_key_false(struct static_key *key) { - if (unlikely(static_key_count(key) > 0)) + if (unlikely_notrace(static_key_count(key) > 0)) return true; return false; } static __always_inline bool static_key_true(struct static_key *key) { - if (likely(static_key_count(key) > 0)) + if (likely_notrace(static_key_count(key) > 0)) return true; return false; } @@ -460,7 +460,7 @@ extern bool ____wrong_branch_error(void); branch = !arch_static_branch_jump(&(x)->key, true); \ else \ branch = ____wrong_branch_error(); \ - likely(branch); \ + likely_notrace(branch); \ }) #define static_branch_unlikely(x) \ @@ -472,13 +472,13 @@ extern bool ____wrong_branch_error(void); branch = arch_static_branch(&(x)->key, false); \ else \ branch = ____wrong_branch_error(); \ - unlikely(branch); \ + unlikely_notrace(branch); \ }) #else /* !CONFIG_JUMP_LABEL */ -#define static_branch_likely(x) likely(static_key_enabled(&(x)->key)) -#define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key)) +#define static_branch_likely(x) likely_notrace(static_key_enabled(&(x)->key)) +#define static_branch_unlikely(x) unlikely_notrace(static_key_enabled(&(x)->key)) #endif /* CONFIG_JUMP_LABEL */ -- cgit v1.2.3 From 997acaf6b4b59c6a9c259740312a69ea549cc684 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 11 Jan 2021 15:37:07 +0000 Subject: lockdep: report broken irq restoration We generally expect local_irq_save() and local_irq_restore() to be paired and sanely nested, and so local_irq_restore() expects to be called with irqs disabled. Thus, within local_irq_restore() we only trace irq flag changes when unmasking irqs. This means that a sequence such as: | local_irq_disable(); | local_irq_save(flags); | local_irq_enable(); | local_irq_restore(flags); ... is liable to break things, as the local_irq_restore() would mask irqs without tracing this change. Similar problems may exist for architectures whose arch_irq_restore() function depends on being called with irqs disabled. We don't consider such sequences to be a good idea, so let's define those as forbidden, and add tooling to detect such broken cases. This patch adds debug code to WARN() when raw_local_irq_restore() is called with irqs enabled. As raw_local_irq_restore() is expected to pair with raw_local_irq_save(), it should never be called with irqs enabled. To avoid the possibility of circular header dependencies between irqflags.h and bug.h, the warning is handled in a separate C file. The new code is all conditional on a new CONFIG_DEBUG_IRQFLAGS symbol which is independent of CONFIG_TRACE_IRQFLAGS. As noted above such cases will confuse lockdep, so CONFIG_DEBUG_LOCKDEP now selects CONFIG_DEBUG_IRQFLAGS. Signed-off-by: Mark Rutland Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20210111153707.10071-1-mark.rutland@arm.com --- include/linux/irqflags.h | 12 ++++++++++++ kernel/locking/Makefile | 1 + kernel/locking/irqflag-debug.c | 11 +++++++++++ lib/Kconfig.debug | 8 ++++++++ 4 files changed, 32 insertions(+) create mode 100644 kernel/locking/irqflag-debug.c (limited to 'include/linux') diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h index 8de0e1373de7..600c10da321a 100644 --- a/include/linux/irqflags.h +++ b/include/linux/irqflags.h @@ -149,6 +149,17 @@ do { \ # define start_critical_timings() do { } while (0) #endif +#ifdef CONFIG_DEBUG_IRQFLAGS +extern void warn_bogus_irq_restore(void); +#define raw_check_bogus_irq_restore() \ + do { \ + if (unlikely(!arch_irqs_disabled())) \ + warn_bogus_irq_restore(); \ + } while (0) +#else +#define raw_check_bogus_irq_restore() do { } while (0) +#endif + /* * Wrap the arch provided IRQ routines to provide appropriate checks. */ @@ -162,6 +173,7 @@ do { \ #define raw_local_irq_restore(flags) \ do { \ typecheck(unsigned long, flags); \ + raw_check_bogus_irq_restore(); \ arch_local_irq_restore(flags); \ } while (0) #define raw_local_save_flags(flags) \ diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile index 6d11cfb9b41f..8838f1d7c4a2 100644 --- a/kernel/locking/Makefile +++ b/kernel/locking/Makefile @@ -15,6 +15,7 @@ CFLAGS_REMOVE_mutex-debug.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_rtmutex-debug.o = $(CC_FLAGS_FTRACE) endif +obj-$(CONFIG_DEBUG_IRQFLAGS) += irqflag-debug.o obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o obj-$(CONFIG_LOCKDEP) += lockdep.o ifeq ($(CONFIG_PROC_FS),y) diff --git a/kernel/locking/irqflag-debug.c b/kernel/locking/irqflag-debug.c new file mode 100644 index 000000000000..9603d207a4ca --- /dev/null +++ b/kernel/locking/irqflag-debug.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include + +void warn_bogus_irq_restore(void) +{ + WARN_ONCE(1, "raw_local_irq_restore() called with IRQs enabled\n"); +} +EXPORT_SYMBOL(warn_bogus_irq_restore); diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index e6e58b26e888..78eadf615df8 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1343,6 +1343,7 @@ config LOCKDEP_SMALL config DEBUG_LOCKDEP bool "Lock dependency engine debugging" depends on DEBUG_KERNEL && LOCKDEP + select DEBUG_IRQFLAGS help If you say Y here, the lock dependency engine will do additional runtime checks to debug itself, at the price @@ -1431,6 +1432,13 @@ config TRACE_IRQFLAGS_NMI depends on TRACE_IRQFLAGS depends on TRACE_IRQFLAGS_NMI_SUPPORT +config DEBUG_IRQFLAGS + bool "Debug IRQ flag manipulation" + help + Enables checks for potentially unsafe enabling or disabling of + interrupts, such as calling raw_local_irq_restore() when interrupts + are enabled. + config STACKTRACE bool "Stack backtrace support" depends on STACKTRACE_SUPPORT -- cgit v1.2.3 From 67e3242ee28052daa90e1fa193efc601939fde58 Mon Sep 17 00:00:00 2001 From: Lina Iyer Date: Wed, 20 Jan 2021 08:50:41 -0700 Subject: PM: domains: inform PM domain of a device's next wakeup Some devices may have a predictable interrupt pattern while executing usecases. An example would be the VSYNC interrupt associated with display devices. A 60 Hz display could cause a interrupt every 16 ms. If the device were in a PM domain, the domain would need to be powered up for device to resume and handle the interrupt. Entering a domain idle state saves power, only if the residency of the idle state is met. Without knowing the idle duration of the domain, the governor would just choose the deepest idle state that matches the QoS requirements. The domain might be powered off just as the device is expecting to wake up. If devices could inform PM frameworks of their next event, the parent PM domain's idle duration can be determined. So let's add the dev_pm_genpd_set_next_wakeup() API for the device to inform PM domains of the impending wakeup. This information will be the domain governor to determine the best idle state given the wakeup. Signed-off-by: Lina Iyer Reviewed-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- drivers/base/power/domain.c | 30 ++++++++++++++++++++++++++++++ include/linux/pm_domain.h | 6 ++++++ 2 files changed, 36 insertions(+) (limited to 'include/linux') diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 9a14eedacb92..014033c7c287 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -423,6 +423,35 @@ int dev_pm_genpd_set_performance_state(struct device *dev, unsigned int state) } EXPORT_SYMBOL_GPL(dev_pm_genpd_set_performance_state); +/** + * dev_pm_genpd_set_next_wakeup - Notify PM framework of an impending wakeup. + * + * @dev: Device to handle + * @next: impending interrupt/wakeup for the device + * + * + * Allow devices to inform of the next wakeup. It's assumed that the users + * guarantee that the genpd wouldn't be detached while this routine is getting + * called. Additionally, it's also assumed that @dev isn't runtime suspended + * (RPM_SUSPENDED)." + * Although devices are expected to update the next_wakeup after the end of + * their usecase as well, it is possible the devices themselves may not know + * about that, so stale @next will be ignored when powering off the domain. + */ +void dev_pm_genpd_set_next_wakeup(struct device *dev, ktime_t next) +{ + struct generic_pm_domain_data *gpd_data; + struct generic_pm_domain *genpd; + + genpd = dev_to_genpd_safe(dev); + if (!genpd) + return; + + gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); + gpd_data->next_wakeup = next; +} +EXPORT_SYMBOL_GPL(dev_pm_genpd_set_next_wakeup); + static int _genpd_power_on(struct generic_pm_domain *genpd, bool timed) { unsigned int state_idx = genpd->state_idx; @@ -1465,6 +1494,7 @@ static struct generic_pm_domain_data *genpd_alloc_dev_data(struct device *dev) gpd_data->td.constraint_changed = true; gpd_data->td.effective_constraint_ns = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS; gpd_data->nb.notifier_call = genpd_dev_pm_qos_notifier; + gpd_data->next_wakeup = KTIME_MAX; spin_lock_irq(&dev->power.lock); diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index 2ca919ae8d36..735583c0bc6d 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -9,6 +9,7 @@ #define _LINUX_PM_DOMAIN_H #include +#include #include #include #include @@ -191,6 +192,7 @@ struct generic_pm_domain_data { struct notifier_block *power_nb; int cpu; unsigned int performance_state; + ktime_t next_wakeup; void *data; }; @@ -217,6 +219,7 @@ int pm_genpd_remove(struct generic_pm_domain *genpd); int dev_pm_genpd_set_performance_state(struct device *dev, unsigned int state); int dev_pm_genpd_add_notifier(struct device *dev, struct notifier_block *nb); int dev_pm_genpd_remove_notifier(struct device *dev); +void dev_pm_genpd_set_next_wakeup(struct device *dev, ktime_t next); extern struct dev_power_governor simple_qos_governor; extern struct dev_power_governor pm_domain_always_on_gov; @@ -275,6 +278,9 @@ static inline int dev_pm_genpd_remove_notifier(struct device *dev) return -EOPNOTSUPP; } +static inline void dev_pm_genpd_set_next_wakeup(struct device *dev, ktime_t next) +{ } + #define simple_qos_governor (*(struct dev_power_governor *)(NULL)) #define pm_domain_always_on_gov (*(struct dev_power_governor *)(NULL)) #endif -- cgit v1.2.3 From c79aa080fb0f60a0e24c87014dc9c2f373e1379b Mon Sep 17 00:00:00 2001 From: Lina Iyer Date: Wed, 20 Jan 2021 08:50:42 -0700 Subject: PM: domains: use device's next wakeup to determine domain idle state Currently, a PM domain's idle state is determined based on whether the QoS requirements are met. However, even entering an idle state may waste power if the minimum residency requirements aren't fulfilled. CPU PM domains use the next timer wakeup for the CPUs in the domain to determine the sleep duration of the domain. This is compared with the idle state residencies to determine the optimal idle state. For other PM domains, determining the sleep length is not that straight forward. But if the device's next_event is available, we can use that to determine the sleep duration of the PM domain. Let's update the domain governor logic to check for idle state residency based on the next wakeup of devices as well as QoS constraints. But since, not all domains may contain devices capable of specifying the next wakeup, let's enable this additional check only if specified by the domain's flags when initializing the domain. Signed-off-by: Lina Iyer Reviewed-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- drivers/base/power/domain_governor.c | 102 +++++++++++++++++++++++++++++++---- include/linux/pm_domain.h | 6 +++ 2 files changed, 99 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/power/domain_governor.c b/drivers/base/power/domain_governor.c index 490ed7deb99a..c6c218758f0b 100644 --- a/drivers/base/power/domain_governor.c +++ b/drivers/base/power/domain_governor.c @@ -117,6 +117,55 @@ static bool default_suspend_ok(struct device *dev) return td->cached_suspend_ok; } +static void update_domain_next_wakeup(struct generic_pm_domain *genpd, ktime_t now) +{ + ktime_t domain_wakeup = KTIME_MAX; + ktime_t next_wakeup; + struct pm_domain_data *pdd; + struct gpd_link *link; + + if (!(genpd->flags & GENPD_FLAG_MIN_RESIDENCY)) + return; + + /* + * Devices that have a predictable wakeup pattern, may specify + * their next wakeup. Let's find the next wakeup from all the + * devices attached to this domain and from all the sub-domains. + * It is possible that component's a next wakeup may have become + * stale when we read that here. We will ignore to ensure the domain + * is able to enter its optimal idle state. + */ + list_for_each_entry(pdd, &genpd->dev_list, list_node) { + next_wakeup = to_gpd_data(pdd)->next_wakeup; + if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now)) + if (ktime_before(next_wakeup, domain_wakeup)) + domain_wakeup = next_wakeup; + } + + list_for_each_entry(link, &genpd->parent_links, parent_node) { + next_wakeup = link->child->next_wakeup; + if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now)) + if (ktime_before(next_wakeup, domain_wakeup)) + domain_wakeup = next_wakeup; + } + + genpd->next_wakeup = domain_wakeup; +} + +static bool next_wakeup_allows_state(struct generic_pm_domain *genpd, + unsigned int state, ktime_t now) +{ + ktime_t domain_wakeup = genpd->next_wakeup; + s64 idle_time_ns, min_sleep_ns; + + min_sleep_ns = genpd->states[state].power_off_latency_ns + + genpd->states[state].residency_ns; + + idle_time_ns = ktime_to_ns(ktime_sub(domain_wakeup, now)); + + return idle_time_ns >= min_sleep_ns; +} + static bool __default_power_down_ok(struct dev_pm_domain *pd, unsigned int state) { @@ -201,16 +250,41 @@ static bool __default_power_down_ok(struct dev_pm_domain *pd, } /** - * default_power_down_ok - Default generic PM domain power off governor routine. + * _default_power_down_ok - Default generic PM domain power off governor routine. * @pd: PM domain to check. * * This routine must be executed under the PM domain's lock. */ -static bool default_power_down_ok(struct dev_pm_domain *pd) +static bool _default_power_down_ok(struct dev_pm_domain *pd, ktime_t now) { struct generic_pm_domain *genpd = pd_to_genpd(pd); + int state_idx = genpd->state_count - 1; struct gpd_link *link; + /* + * Find the next wakeup from devices that can determine their own wakeup + * to find when the domain would wakeup and do it for every device down + * the hierarchy. It is not worth while to sleep if the state's residency + * cannot be met. + */ + update_domain_next_wakeup(genpd, now); + if ((genpd->flags & GENPD_FLAG_MIN_RESIDENCY) && (genpd->next_wakeup != KTIME_MAX)) { + /* Let's find out the deepest domain idle state, the devices prefer */ + while (state_idx >= 0) { + if (next_wakeup_allows_state(genpd, state_idx, now)) { + genpd->max_off_time_changed = true; + break; + } + state_idx--; + } + + if (state_idx < 0) { + state_idx = 0; + genpd->cached_power_down_ok = false; + goto done; + } + } + if (!genpd->max_off_time_changed) { genpd->state_idx = genpd->cached_power_down_state_idx; return genpd->cached_power_down_ok; @@ -228,21 +302,30 @@ static bool default_power_down_ok(struct dev_pm_domain *pd) genpd->max_off_time_ns = -1; genpd->max_off_time_changed = false; genpd->cached_power_down_ok = true; - genpd->state_idx = genpd->state_count - 1; - /* Find a state to power down to, starting from the deepest. */ - while (!__default_power_down_ok(pd, genpd->state_idx)) { - if (genpd->state_idx == 0) { + /* + * Find a state to power down to, starting from the state + * determined by the next wakeup. + */ + while (!__default_power_down_ok(pd, state_idx)) { + if (state_idx == 0) { genpd->cached_power_down_ok = false; break; } - genpd->state_idx--; + state_idx--; } +done: + genpd->state_idx = state_idx; genpd->cached_power_down_state_idx = genpd->state_idx; return genpd->cached_power_down_ok; } +static bool default_power_down_ok(struct dev_pm_domain *pd) +{ + return _default_power_down_ok(pd, ktime_get()); +} + static bool always_on_power_down_ok(struct dev_pm_domain *domain) { return false; @@ -254,11 +337,12 @@ static bool cpu_power_down_ok(struct dev_pm_domain *pd) struct generic_pm_domain *genpd = pd_to_genpd(pd); struct cpuidle_device *dev; ktime_t domain_wakeup, next_hrtimer; + ktime_t now = ktime_get(); s64 idle_duration_ns; int cpu, i; /* Validate dev PM QoS constraints. */ - if (!default_power_down_ok(pd)) + if (!_default_power_down_ok(pd, now)) return false; if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN)) @@ -280,7 +364,7 @@ static bool cpu_power_down_ok(struct dev_pm_domain *pd) } /* The minimum idle duration is from now - until the next wakeup. */ - idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, ktime_get())); + idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, now)); if (idle_duration_ns <= 0) return false; diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index 735583c0bc6d..dfcfbcecc34b 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -56,6 +56,10 @@ * * GENPD_FLAG_RPM_ALWAYS_ON: Instructs genpd to always keep the PM domain * powered on except for system suspend. + * + * GENPD_FLAG_MIN_RESIDENCY: Enable the genpd governor to consider its + * components' next wakeup when determining the + * optimal idle state. */ #define GENPD_FLAG_PM_CLK (1U << 0) #define GENPD_FLAG_IRQ_SAFE (1U << 1) @@ -63,6 +67,7 @@ #define GENPD_FLAG_ACTIVE_WAKEUP (1U << 3) #define GENPD_FLAG_CPU_DOMAIN (1U << 4) #define GENPD_FLAG_RPM_ALWAYS_ON (1U << 5) +#define GENPD_FLAG_MIN_RESIDENCY (1U << 6) enum gpd_status { GENPD_STATE_ON = 0, /* PM domain is on */ @@ -130,6 +135,7 @@ struct generic_pm_domain { unsigned int state); struct gpd_dev_ops dev_ops; s64 max_off_time_ns; /* Maximum allowed "suspended" time. */ + ktime_t next_wakeup; /* Maintained by the domain governor */ bool max_off_time_changed; bool cached_power_down_ok; bool cached_power_down_state_idx; -- cgit v1.2.3 From f3196bb0f14c0ffb5089c15668bda196c98d3900 Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Fri, 11 Dec 2020 22:12:16 -0800 Subject: net/mlx5: Introduce vhca state event notifier vhca state events indicates change in the state of the vhca that may occur due to a SF allocation, deallocation or enabling/disabling the SF HCA. Introduce vhca state event handler which will be used by SF devlink port manager and SF hardware id allocator in subsequent patches to act on the event. This enables single entity to subscribe, query and rearm the event for a function. Signed-off-by: Parav Pandit Reviewed-by: Vu Pham Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/Kconfig | 9 + drivers/net/ethernet/mellanox/mlx5/core/Makefile | 4 + drivers/net/ethernet/mellanox/mlx5/core/cmd.c | 4 + drivers/net/ethernet/mellanox/mlx5/core/eq.c | 3 + drivers/net/ethernet/mellanox/mlx5/core/events.c | 7 + drivers/net/ethernet/mellanox/mlx5/core/main.c | 16 ++ .../net/ethernet/mellanox/mlx5/core/mlx5_core.h | 2 + .../mellanox/mlx5/core/sf/mlx5_ifc_vhca_event.h | 82 +++++++++ drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h | 45 +++++ .../ethernet/mellanox/mlx5/core/sf/vhca_event.c | 189 +++++++++++++++++++++ .../ethernet/mellanox/mlx5/core/sf/vhca_event.h | 57 +++++++ include/linux/mlx5/driver.h | 4 + 12 files changed, 422 insertions(+) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sf/mlx5_ifc_vhca_event.h create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sf/vhca_event.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sf/vhca_event.h (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig index 6e4d7bb7fea2..d6c48582e7a8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig +++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig @@ -203,3 +203,12 @@ config MLX5_SW_STEERING default y help Build support for software-managed steering in the NIC. + +config MLX5_SF + bool "Mellanox Technologies subfunction device support using auxiliary device" + depends on MLX5_CORE && MLX5_CORE_EN + default n + help + Build support for subfuction device in the NIC. A Mellanox subfunction + device can support RDMA, netdevice and vdpa device. + It is similar to a SRIOV VF but it doesn't require SRIOV support. diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 134bd038ae8a..22ef2ebbee96 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -86,3 +86,7 @@ mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o steering/dr_ste_v0.o \ steering/dr_cmd.o steering/dr_fw.o \ steering/dr_action.o steering/fs_dr.o +# +# SF device +# +mlx5_core-$(CONFIG_MLX5_SF) += sf/vhca_event.o diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index 50c7b9ee80c3..47dcc3ac2cf0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -464,6 +464,8 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, case MLX5_CMD_OP_ALLOC_MEMIC: case MLX5_CMD_OP_MODIFY_XRQ: case MLX5_CMD_OP_RELEASE_XRQ_ERROR: + case MLX5_CMD_OP_QUERY_VHCA_STATE: + case MLX5_CMD_OP_MODIFY_VHCA_STATE: *status = MLX5_DRIVER_STATUS_ABORTED; *synd = MLX5_DRIVER_SYND; return -EIO; @@ -657,6 +659,8 @@ const char *mlx5_command_str(int command) MLX5_COMMAND_STR_CASE(DESTROY_UMEM); MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR); MLX5_COMMAND_STR_CASE(MODIFY_XRQ); + MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE); + MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE); default: return "unknown command opcode"; } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index fc0afa03d407..421febebc658 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -595,6 +595,9 @@ static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4]) async_event_mask |= (1ull << MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED); + if (MLX5_CAP_GEN_MAX(dev, vhca_state)) + async_event_mask |= (1ull << MLX5_EVENT_TYPE_VHCA_STATE_CHANGE); + mask[0] = async_event_mask; if (MLX5_CAP_GEN(dev, event_cap)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/events.c b/drivers/net/ethernet/mellanox/mlx5/core/events.c index 3ce17c3d7a00..5523d218e5fb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/events.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/events.c @@ -110,6 +110,8 @@ static const char *eqe_type_str(u8 type) return "MLX5_EVENT_TYPE_CMD"; case MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED: return "MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED"; + case MLX5_EVENT_TYPE_VHCA_STATE_CHANGE: + return "MLX5_EVENT_TYPE_VHCA_STATE_CHANGE"; case MLX5_EVENT_TYPE_PAGE_REQUEST: return "MLX5_EVENT_TYPE_PAGE_REQUEST"; case MLX5_EVENT_TYPE_PAGE_FAULT: @@ -403,3 +405,8 @@ int mlx5_notifier_call_chain(struct mlx5_events *events, unsigned int event, voi { return atomic_notifier_call_chain(&events->nh, event, data); } + +void mlx5_events_work_enqueue(struct mlx5_core_dev *dev, struct work_struct *work) +{ + queue_work(dev->priv.events->wq, work); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index ca6f2fc39ea0..b16f57befe52 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -73,6 +73,7 @@ #include "ecpf.h" #include "lib/hv_vhca.h" #include "diag/rsc_dump.h" +#include "sf/vhca_event.h" MODULE_AUTHOR("Eli Cohen "); MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver"); @@ -567,6 +568,8 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) if (MLX5_CAP_GEN_MAX(dev, mkey_by_name)) MLX5_SET(cmd_hca_cap, set_hca_cap, mkey_by_name, 1); + mlx5_vhca_state_cap_handle(dev, set_hca_cap); + return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); } @@ -884,6 +887,12 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) goto err_eswitch_cleanup; } + err = mlx5_vhca_event_init(dev); + if (err) { + mlx5_core_err(dev, "Failed to init vhca event notifier %d\n", err); + goto err_fpga_cleanup; + } + dev->dm = mlx5_dm_create(dev); if (IS_ERR(dev->dm)) mlx5_core_warn(dev, "Failed to init device memory%d\n", err); @@ -894,6 +903,8 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) return 0; +err_fpga_cleanup: + mlx5_fpga_cleanup(dev); err_eswitch_cleanup: mlx5_eswitch_cleanup(dev->priv.eswitch); err_sriov_cleanup: @@ -925,6 +936,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev) mlx5_hv_vhca_destroy(dev->hv_vhca); mlx5_fw_tracer_destroy(dev->tracer); mlx5_dm_cleanup(dev); + mlx5_vhca_event_cleanup(dev); mlx5_fpga_cleanup(dev); mlx5_eswitch_cleanup(dev->priv.eswitch); mlx5_sriov_cleanup(dev); @@ -1129,6 +1141,8 @@ static int mlx5_load(struct mlx5_core_dev *dev) goto err_sriov; } + mlx5_vhca_event_start(dev); + err = mlx5_ec_init(dev); if (err) { mlx5_core_err(dev, "Failed to init embedded CPU\n"); @@ -1146,6 +1160,7 @@ static int mlx5_load(struct mlx5_core_dev *dev) err_sriov: mlx5_ec_cleanup(dev); err_ec: + mlx5_vhca_event_stop(dev); mlx5_cleanup_fs(dev); err_fs: mlx5_accel_tls_cleanup(dev); @@ -1173,6 +1188,7 @@ static void mlx5_unload(struct mlx5_core_dev *dev) { mlx5_sriov_detach(dev); mlx5_ec_cleanup(dev); + mlx5_vhca_event_stop(dev); mlx5_cleanup_fs(dev); mlx5_accel_ipsec_cleanup(dev); mlx5_accel_tls_cleanup(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 0a0302ce7144..a33b7496d748 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -259,4 +259,6 @@ void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state); void mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup); int mlx5_load_one(struct mlx5_core_dev *dev, bool boot); + +void mlx5_events_work_enqueue(struct mlx5_core_dev *dev, struct work_struct *work); #endif /* __MLX5_CORE_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/mlx5_ifc_vhca_event.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/mlx5_ifc_vhca_event.h new file mode 100644 index 000000000000..1daf5a122ba3 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/mlx5_ifc_vhca_event.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020 Mellanox Technologies Ltd */ + +#ifndef __MLX5_IFC_VHCA_EVENT_H__ +#define __MLX5_IFC_VHCA_EVENT_H__ + +enum mlx5_ifc_vhca_state { + MLX5_VHCA_STATE_INVALID = 0x0, + MLX5_VHCA_STATE_ALLOCATED = 0x1, + MLX5_VHCA_STATE_ACTIVE = 0x2, + MLX5_VHCA_STATE_IN_USE = 0x3, + MLX5_VHCA_STATE_TEARDOWN_REQUEST = 0x4, +}; + +struct mlx5_ifc_vhca_state_context_bits { + u8 arm_change_event[0x1]; + u8 reserved_at_1[0xb]; + u8 vhca_state[0x4]; + u8 reserved_at_10[0x10]; + + u8 sw_function_id[0x20]; + + u8 reserved_at_40[0x80]; +}; + +struct mlx5_ifc_query_vhca_state_out_bits { + u8 status[0x8]; + u8 reserved_at_8[0x18]; + + u8 syndrome[0x20]; + + u8 reserved_at_40[0x40]; + + struct mlx5_ifc_vhca_state_context_bits vhca_state_context; +}; + +struct mlx5_ifc_query_vhca_state_in_bits { + u8 opcode[0x10]; + u8 uid[0x10]; + + u8 reserved_at_20[0x10]; + u8 op_mod[0x10]; + + u8 embedded_cpu_function[0x1]; + u8 reserved_at_41[0xf]; + u8 function_id[0x10]; + + u8 reserved_at_60[0x20]; +}; + +struct mlx5_ifc_vhca_state_field_select_bits { + u8 reserved_at_0[0x1e]; + u8 sw_function_id[0x1]; + u8 arm_change_event[0x1]; +}; + +struct mlx5_ifc_modify_vhca_state_out_bits { + u8 status[0x8]; + u8 reserved_at_8[0x18]; + + u8 syndrome[0x20]; + + u8 reserved_at_40[0x40]; +}; + +struct mlx5_ifc_modify_vhca_state_in_bits { + u8 opcode[0x10]; + u8 uid[0x10]; + + u8 reserved_at_20[0x10]; + u8 op_mod[0x10]; + + u8 embedded_cpu_function[0x1]; + u8 reserved_at_41[0xf]; + u8 function_id[0x10]; + + struct mlx5_ifc_vhca_state_field_select_bits vhca_state_field_select; + + struct mlx5_ifc_vhca_state_context_bits vhca_state_context; +}; + +#endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h new file mode 100644 index 000000000000..623191679b49 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020 Mellanox Technologies Ltd */ + +#ifndef __MLX5_SF_H__ +#define __MLX5_SF_H__ + +#include + +static inline u16 mlx5_sf_start_function_id(const struct mlx5_core_dev *dev) +{ + return MLX5_CAP_GEN(dev, sf_base_id); +} + +#ifdef CONFIG_MLX5_SF + +static inline bool mlx5_sf_supported(const struct mlx5_core_dev *dev) +{ + return MLX5_CAP_GEN(dev, sf); +} + +static inline u16 mlx5_sf_max_functions(const struct mlx5_core_dev *dev) +{ + if (!mlx5_sf_supported(dev)) + return 0; + if (MLX5_CAP_GEN(dev, max_num_sf)) + return MLX5_CAP_GEN(dev, max_num_sf); + else + return 1 << MLX5_CAP_GEN(dev, log_max_sf); +} + +#else + +static inline bool mlx5_sf_supported(const struct mlx5_core_dev *dev) +{ + return false; +} + +static inline u16 mlx5_sf_max_functions(const struct mlx5_core_dev *dev) +{ + return 0; +} + +#endif + +#endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/vhca_event.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/vhca_event.c new file mode 100644 index 000000000000..af2f2dd9db25 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/vhca_event.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020 Mellanox Technologies Ltd */ + +#include +#include "mlx5_ifc_vhca_event.h" +#include "mlx5_core.h" +#include "vhca_event.h" +#include "ecpf.h" + +struct mlx5_vhca_state_notifier { + struct mlx5_core_dev *dev; + struct mlx5_nb nb; + struct blocking_notifier_head n_head; +}; + +struct mlx5_vhca_event_work { + struct work_struct work; + struct mlx5_vhca_state_notifier *notifier; + struct mlx5_vhca_state_event event; +}; + +int mlx5_cmd_query_vhca_state(struct mlx5_core_dev *dev, u16 function_id, + bool ecpu, u32 *out, u32 outlen) +{ + u32 in[MLX5_ST_SZ_DW(query_vhca_state_in)] = {}; + + MLX5_SET(query_vhca_state_in, in, opcode, MLX5_CMD_OP_QUERY_VHCA_STATE); + MLX5_SET(query_vhca_state_in, in, function_id, function_id); + MLX5_SET(query_vhca_state_in, in, embedded_cpu_function, ecpu); + + return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen); +} + +static int mlx5_cmd_modify_vhca_state(struct mlx5_core_dev *dev, u16 function_id, + bool ecpu, u32 *in, u32 inlen) +{ + u32 out[MLX5_ST_SZ_DW(modify_vhca_state_out)] = {}; + + MLX5_SET(modify_vhca_state_in, in, opcode, MLX5_CMD_OP_MODIFY_VHCA_STATE); + MLX5_SET(modify_vhca_state_in, in, function_id, function_id); + MLX5_SET(modify_vhca_state_in, in, embedded_cpu_function, ecpu); + + return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); +} + +int mlx5_modify_vhca_sw_id(struct mlx5_core_dev *dev, u16 function_id, bool ecpu, u32 sw_fn_id) +{ + u32 out[MLX5_ST_SZ_DW(modify_vhca_state_out)] = {}; + u32 in[MLX5_ST_SZ_DW(modify_vhca_state_in)] = {}; + + MLX5_SET(modify_vhca_state_in, in, opcode, MLX5_CMD_OP_MODIFY_VHCA_STATE); + MLX5_SET(modify_vhca_state_in, in, function_id, function_id); + MLX5_SET(modify_vhca_state_in, in, embedded_cpu_function, ecpu); + MLX5_SET(modify_vhca_state_in, in, vhca_state_field_select.sw_function_id, 1); + MLX5_SET(modify_vhca_state_in, in, vhca_state_context.sw_function_id, sw_fn_id); + + return mlx5_cmd_exec_inout(dev, modify_vhca_state, in, out); +} + +int mlx5_vhca_event_arm(struct mlx5_core_dev *dev, u16 function_id, bool ecpu) +{ + u32 in[MLX5_ST_SZ_DW(modify_vhca_state_in)] = {}; + + MLX5_SET(modify_vhca_state_in, in, vhca_state_context.arm_change_event, 1); + MLX5_SET(modify_vhca_state_in, in, vhca_state_field_select.arm_change_event, 1); + + return mlx5_cmd_modify_vhca_state(dev, function_id, ecpu, in, sizeof(in)); +} + +static void +mlx5_vhca_event_notify(struct mlx5_core_dev *dev, struct mlx5_vhca_state_event *event) +{ + u32 out[MLX5_ST_SZ_DW(query_vhca_state_out)] = {}; + int err; + + err = mlx5_cmd_query_vhca_state(dev, event->function_id, event->ecpu, out, sizeof(out)); + if (err) + return; + + event->sw_function_id = MLX5_GET(query_vhca_state_out, out, + vhca_state_context.sw_function_id); + event->new_vhca_state = MLX5_GET(query_vhca_state_out, out, + vhca_state_context.vhca_state); + + mlx5_vhca_event_arm(dev, event->function_id, event->ecpu); + + blocking_notifier_call_chain(&dev->priv.vhca_state_notifier->n_head, 0, event); +} + +static void mlx5_vhca_state_work_handler(struct work_struct *_work) +{ + struct mlx5_vhca_event_work *work = container_of(_work, struct mlx5_vhca_event_work, work); + struct mlx5_vhca_state_notifier *notifier = work->notifier; + struct mlx5_core_dev *dev = notifier->dev; + + mlx5_vhca_event_notify(dev, &work->event); +} + +static int +mlx5_vhca_state_change_notifier(struct notifier_block *nb, unsigned long type, void *data) +{ + struct mlx5_vhca_state_notifier *notifier = + mlx5_nb_cof(nb, struct mlx5_vhca_state_notifier, nb); + struct mlx5_vhca_event_work *work; + struct mlx5_eqe *eqe = data; + + work = kzalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return NOTIFY_DONE; + INIT_WORK(&work->work, &mlx5_vhca_state_work_handler); + work->notifier = notifier; + work->event.function_id = be16_to_cpu(eqe->data.vhca_state.function_id); + work->event.ecpu = be16_to_cpu(eqe->data.vhca_state.ec_function); + mlx5_events_work_enqueue(notifier->dev, &work->work); + return NOTIFY_OK; +} + +void mlx5_vhca_state_cap_handle(struct mlx5_core_dev *dev, void *set_hca_cap) +{ + if (!mlx5_vhca_event_supported(dev)) + return; + + MLX5_SET(cmd_hca_cap, set_hca_cap, vhca_state, 1); + MLX5_SET(cmd_hca_cap, set_hca_cap, event_on_vhca_state_allocated, 1); + MLX5_SET(cmd_hca_cap, set_hca_cap, event_on_vhca_state_active, 1); + MLX5_SET(cmd_hca_cap, set_hca_cap, event_on_vhca_state_in_use, 1); + MLX5_SET(cmd_hca_cap, set_hca_cap, event_on_vhca_state_teardown_request, 1); +} + +int mlx5_vhca_event_init(struct mlx5_core_dev *dev) +{ + struct mlx5_vhca_state_notifier *notifier; + + if (!mlx5_vhca_event_supported(dev)) + return 0; + + notifier = kzalloc(sizeof(*notifier), GFP_KERNEL); + if (!notifier) + return -ENOMEM; + + dev->priv.vhca_state_notifier = notifier; + notifier->dev = dev; + BLOCKING_INIT_NOTIFIER_HEAD(¬ifier->n_head); + MLX5_NB_INIT(¬ifier->nb, mlx5_vhca_state_change_notifier, VHCA_STATE_CHANGE); + return 0; +} + +void mlx5_vhca_event_cleanup(struct mlx5_core_dev *dev) +{ + if (!mlx5_vhca_event_supported(dev)) + return; + + kfree(dev->priv.vhca_state_notifier); + dev->priv.vhca_state_notifier = NULL; +} + +void mlx5_vhca_event_start(struct mlx5_core_dev *dev) +{ + struct mlx5_vhca_state_notifier *notifier; + + if (!dev->priv.vhca_state_notifier) + return; + + notifier = dev->priv.vhca_state_notifier; + mlx5_eq_notifier_register(dev, ¬ifier->nb); +} + +void mlx5_vhca_event_stop(struct mlx5_core_dev *dev) +{ + struct mlx5_vhca_state_notifier *notifier; + + if (!dev->priv.vhca_state_notifier) + return; + + notifier = dev->priv.vhca_state_notifier; + mlx5_eq_notifier_unregister(dev, ¬ifier->nb); +} + +int mlx5_vhca_event_notifier_register(struct mlx5_core_dev *dev, struct notifier_block *nb) +{ + if (!dev->priv.vhca_state_notifier) + return -EOPNOTSUPP; + return blocking_notifier_chain_register(&dev->priv.vhca_state_notifier->n_head, nb); +} + +void mlx5_vhca_event_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *nb) +{ + blocking_notifier_chain_unregister(&dev->priv.vhca_state_notifier->n_head, nb); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/vhca_event.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/vhca_event.h new file mode 100644 index 000000000000..1fe1ec6f4d4b --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/vhca_event.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020 Mellanox Technologies Ltd */ + +#ifndef __MLX5_VHCA_EVENT_H__ +#define __MLX5_VHCA_EVENT_H__ + +#ifdef CONFIG_MLX5_SF + +struct mlx5_vhca_state_event { + u16 function_id; + u16 sw_function_id; + u8 new_vhca_state; + bool ecpu; +}; + +static inline bool mlx5_vhca_event_supported(const struct mlx5_core_dev *dev) +{ + return MLX5_CAP_GEN_MAX(dev, vhca_state); +} + +void mlx5_vhca_state_cap_handle(struct mlx5_core_dev *dev, void *set_hca_cap); +int mlx5_vhca_event_init(struct mlx5_core_dev *dev); +void mlx5_vhca_event_cleanup(struct mlx5_core_dev *dev); +void mlx5_vhca_event_start(struct mlx5_core_dev *dev); +void mlx5_vhca_event_stop(struct mlx5_core_dev *dev); +int mlx5_vhca_event_notifier_register(struct mlx5_core_dev *dev, struct notifier_block *nb); +void mlx5_vhca_event_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *nb); +int mlx5_modify_vhca_sw_id(struct mlx5_core_dev *dev, u16 function_id, bool ecpu, u32 sw_fn_id); +int mlx5_vhca_event_arm(struct mlx5_core_dev *dev, u16 function_id, bool ecpu); +int mlx5_cmd_query_vhca_state(struct mlx5_core_dev *dev, u16 function_id, + bool ecpu, u32 *out, u32 outlen); +#else + +static inline void mlx5_vhca_state_cap_handle(struct mlx5_core_dev *dev, void *set_hca_cap) +{ +} + +static inline int mlx5_vhca_event_init(struct mlx5_core_dev *dev) +{ + return 0; +} + +static inline void mlx5_vhca_event_cleanup(struct mlx5_core_dev *dev) +{ +} + +static inline void mlx5_vhca_event_start(struct mlx5_core_dev *dev) +{ +} + +static inline void mlx5_vhca_event_stop(struct mlx5_core_dev *dev) +{ +} + +#endif + +#endif diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index f93bfe7473aa..ffba0786051e 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -507,6 +507,7 @@ struct mlx5_devcom; struct mlx5_fw_reset; struct mlx5_eq_table; struct mlx5_irq_table; +struct mlx5_vhca_state_notifier; struct mlx5_rate_limit { u32 rate; @@ -603,6 +604,9 @@ struct mlx5_priv { struct mlx5_bfreg_data bfregs; struct mlx5_uars_page *uar; +#ifdef CONFIG_MLX5_SF + struct mlx5_vhca_state_notifier *vhca_state_notifier; +#endif }; enum mlx5_device_state { -- cgit v1.2.3 From 90d010b8634b89a97ca3b7aa6a88fd566fc77717 Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Fri, 11 Dec 2020 22:12:17 -0800 Subject: net/mlx5: SF, Add auxiliary device support Introduce API to add and delete an auxiliary device for an SF. Each SF has its own dedicated window in the PCI BAR 2. SF device is similar to PCI PF and VF that supports multiple class of devices such as net, rdma and vdpa. SF device will be added or removed in subsequent patch during SF devlink port function state change command. A subfunction device exposes user supplied subfunction number which will be further used by systemd/udev to have deterministic name for its netdevice and rdma device. An mlx5 subfunction auxiliary device example: $ devlink dev eswitch set pci/0000:06:00.0 mode switchdev $ devlink port show pci/0000:06:00.0/65535: type eth netdev ens2f0np0 flavour physical port 0 splittable false $ devlink port add pci/0000:06:00.0 flavour pcisf pfnum 0 sfnum 88 pci/0000:08:00.0/32768: type eth netdev eth6 flavour pcisf controller 0 pfnum 0 sfnum 88 external false splittable false function: hw_addr 00:00:00:00:00:00 state inactive opstate detached $ devlink port show ens2f0npf0sf88 pci/0000:06:00.0/32768: type eth netdev ens2f0npf0sf88 flavour pcisf controller 0 pfnum 0 sfnum 88 external false splittable false function: hw_addr 00:00:00:00:88:88 state inactive opstate detached $ devlink port function set ens2f0npf0sf88 hw_addr 00:00:00:00:88:88 state active On activation, $ ls -l /sys/bus/auxiliary/devices/ mlx5_core.sf.4 -> ../../../devices/pci0000:00/0000:00:03.0/0000:06:00.0/mlx5_core.sf.4 $ cat /sys/bus/auxiliary/devices/mlx5_core.sf.4/sfnum 88 Signed-off-by: Parav Pandit Reviewed-by: Vu Pham Signed-off-by: Saeed Mahameed --- .../device_drivers/ethernet/mellanox/mlx5.rst | 5 + drivers/net/ethernet/mellanox/mlx5/core/Makefile | 2 +- drivers/net/ethernet/mellanox/mlx5/core/main.c | 4 + .../net/ethernet/mellanox/mlx5/core/sf/dev/dev.c | 265 +++++++++++++++++++++ .../net/ethernet/mellanox/mlx5/core/sf/dev/dev.h | 35 +++ include/linux/mlx5/driver.h | 2 + 6 files changed, 312 insertions(+), 1 deletion(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h (limited to 'include/linux') diff --git a/Documentation/networking/device_drivers/ethernet/mellanox/mlx5.rst b/Documentation/networking/device_drivers/ethernet/mellanox/mlx5.rst index e9b65035cd47..a5eb22793bb9 100644 --- a/Documentation/networking/device_drivers/ethernet/mellanox/mlx5.rst +++ b/Documentation/networking/device_drivers/ethernet/mellanox/mlx5.rst @@ -97,6 +97,11 @@ Enabling the driver and kconfig options | Provides low-level InfiniBand/RDMA and `RoCE `_ support. +**CONFIG_MLX5_SF=(y/n)** + +| Build support for subfunction. +| Subfunctons are more light weight than PCI SRIOV VFs. Choosing this option +| will enable support for creating subfunction devices. **External options** ( Choose if the corresponding mlx5 feature is required ) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 22ef2ebbee96..f5b2e9101348 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -89,4 +89,4 @@ mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o # # SF device # -mlx5_core-$(CONFIG_MLX5_SF) += sf/vhca_event.o +mlx5_core-$(CONFIG_MLX5_SF) += sf/vhca_event.o sf/dev/dev.o diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index b16f57befe52..26b5502712a6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -74,6 +74,7 @@ #include "lib/hv_vhca.h" #include "diag/rsc_dump.h" #include "sf/vhca_event.h" +#include "sf/dev/dev.h" MODULE_AUTHOR("Eli Cohen "); MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver"); @@ -1155,6 +1156,8 @@ static int mlx5_load(struct mlx5_core_dev *dev) goto err_sriov; } + mlx5_sf_dev_table_create(dev); + return 0; err_sriov: @@ -1186,6 +1189,7 @@ err_irq_table: static void mlx5_unload(struct mlx5_core_dev *dev) { + mlx5_sf_dev_table_destroy(dev); mlx5_sriov_detach(dev); mlx5_ec_cleanup(dev); mlx5_vhca_event_stop(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c new file mode 100644 index 000000000000..4a8eeb7c853e --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020 Mellanox Technologies Ltd */ + +#include +#include +#include "mlx5_core.h" +#include "dev.h" +#include "sf/vhca_event.h" +#include "sf/sf.h" +#include "sf/mlx5_ifc_vhca_event.h" +#include "ecpf.h" + +struct mlx5_sf_dev_table { + struct xarray devices; + unsigned int max_sfs; + phys_addr_t base_address; + u64 sf_bar_length; + struct notifier_block nb; + struct mlx5_core_dev *dev; +}; + +static bool mlx5_sf_dev_supported(const struct mlx5_core_dev *dev) +{ + return MLX5_CAP_GEN(dev, sf) && mlx5_vhca_event_supported(dev); +} + +static ssize_t sfnum_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct auxiliary_device *adev = container_of(dev, struct auxiliary_device, dev); + struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev); + + return scnprintf(buf, PAGE_SIZE, "%u\n", sf_dev->sfnum); +} +static DEVICE_ATTR_RO(sfnum); + +static struct attribute *sf_device_attrs[] = { + &dev_attr_sfnum.attr, + NULL, +}; + +static const struct attribute_group sf_attr_group = { + .attrs = sf_device_attrs, +}; + +static const struct attribute_group *sf_attr_groups[2] = { + &sf_attr_group, + NULL +}; + +static void mlx5_sf_dev_release(struct device *device) +{ + struct auxiliary_device *adev = container_of(device, struct auxiliary_device, dev); + struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev); + + mlx5_adev_idx_free(adev->id); + kfree(sf_dev); +} + +static void mlx5_sf_dev_remove(struct mlx5_sf_dev *sf_dev) +{ + auxiliary_device_delete(&sf_dev->adev); + auxiliary_device_uninit(&sf_dev->adev); +} + +static void mlx5_sf_dev_add(struct mlx5_core_dev *dev, u16 sf_index, u32 sfnum) +{ + struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table; + struct mlx5_sf_dev *sf_dev; + struct pci_dev *pdev; + int err; + int id; + + id = mlx5_adev_idx_alloc(); + if (id < 0) { + err = id; + goto add_err; + } + + sf_dev = kzalloc(sizeof(*sf_dev), GFP_KERNEL); + if (!sf_dev) { + mlx5_adev_idx_free(id); + err = -ENOMEM; + goto add_err; + } + pdev = dev->pdev; + sf_dev->adev.id = id; + sf_dev->adev.name = MLX5_SF_DEV_ID_NAME; + sf_dev->adev.dev.release = mlx5_sf_dev_release; + sf_dev->adev.dev.parent = &pdev->dev; + sf_dev->adev.dev.groups = sf_attr_groups; + sf_dev->sfnum = sfnum; + sf_dev->parent_mdev = dev; + + if (!table->max_sfs) { + mlx5_adev_idx_free(id); + kfree(sf_dev); + err = -EOPNOTSUPP; + goto add_err; + } + sf_dev->bar_base_addr = table->base_address + (sf_index * table->sf_bar_length); + + err = auxiliary_device_init(&sf_dev->adev); + if (err) { + mlx5_adev_idx_free(id); + kfree(sf_dev); + goto add_err; + } + + err = auxiliary_device_add(&sf_dev->adev); + if (err) { + put_device(&sf_dev->adev.dev); + goto add_err; + } + + err = xa_insert(&table->devices, sf_index, sf_dev, GFP_KERNEL); + if (err) + goto xa_err; + return; + +xa_err: + mlx5_sf_dev_remove(sf_dev); +add_err: + mlx5_core_err(dev, "SF DEV: fail device add for index=%d sfnum=%d err=%d\n", + sf_index, sfnum, err); +} + +static void mlx5_sf_dev_del(struct mlx5_core_dev *dev, struct mlx5_sf_dev *sf_dev, u16 sf_index) +{ + struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table; + + xa_erase(&table->devices, sf_index); + mlx5_sf_dev_remove(sf_dev); +} + +static int +mlx5_sf_dev_state_change_handler(struct notifier_block *nb, unsigned long event_code, void *data) +{ + struct mlx5_sf_dev_table *table = container_of(nb, struct mlx5_sf_dev_table, nb); + const struct mlx5_vhca_state_event *event = data; + struct mlx5_sf_dev *sf_dev; + u16 sf_index; + + sf_index = event->function_id - MLX5_CAP_GEN(table->dev, sf_base_id); + sf_dev = xa_load(&table->devices, sf_index); + switch (event->new_vhca_state) { + case MLX5_VHCA_STATE_ALLOCATED: + if (sf_dev) + mlx5_sf_dev_del(table->dev, sf_dev, sf_index); + break; + case MLX5_VHCA_STATE_TEARDOWN_REQUEST: + if (sf_dev) + mlx5_sf_dev_del(table->dev, sf_dev, sf_index); + else + mlx5_core_err(table->dev, + "SF DEV: teardown state for invalid dev index=%d fn_id=0x%x\n", + sf_index, event->sw_function_id); + break; + case MLX5_VHCA_STATE_ACTIVE: + if (!sf_dev) + mlx5_sf_dev_add(table->dev, sf_index, event->sw_function_id); + break; + default: + break; + } + return 0; +} + +static int mlx5_sf_dev_vhca_arm_all(struct mlx5_sf_dev_table *table) +{ + struct mlx5_core_dev *dev = table->dev; + u16 max_functions; + u16 function_id; + int err = 0; + bool ecpu; + int i; + + max_functions = mlx5_sf_max_functions(dev); + function_id = MLX5_CAP_GEN(dev, sf_base_id); + ecpu = mlx5_read_embedded_cpu(dev); + /* Arm the vhca context as the vhca event notifier */ + for (i = 0; i < max_functions; i++) { + err = mlx5_vhca_event_arm(dev, function_id, ecpu); + if (err) + return err; + + function_id++; + } + return 0; +} + +void mlx5_sf_dev_table_create(struct mlx5_core_dev *dev) +{ + struct mlx5_sf_dev_table *table; + unsigned int max_sfs; + int err; + + if (!mlx5_sf_dev_supported(dev) || !mlx5_vhca_event_supported(dev)) + return; + + table = kzalloc(sizeof(*table), GFP_KERNEL); + if (!table) { + err = -ENOMEM; + goto table_err; + } + + table->nb.notifier_call = mlx5_sf_dev_state_change_handler; + table->dev = dev; + if (MLX5_CAP_GEN(dev, max_num_sf)) + max_sfs = MLX5_CAP_GEN(dev, max_num_sf); + else + max_sfs = 1 << MLX5_CAP_GEN(dev, log_max_sf); + table->sf_bar_length = 1 << (MLX5_CAP_GEN(dev, log_min_sf_size) + 12); + table->base_address = pci_resource_start(dev->pdev, 2); + table->max_sfs = max_sfs; + xa_init(&table->devices); + dev->priv.sf_dev_table = table; + + err = mlx5_vhca_event_notifier_register(dev, &table->nb); + if (err) + goto vhca_err; + err = mlx5_sf_dev_vhca_arm_all(table); + if (err) + goto arm_err; + mlx5_core_dbg(dev, "SF DEV: max sf devices=%d\n", max_sfs); + return; + +arm_err: + mlx5_vhca_event_notifier_unregister(dev, &table->nb); +vhca_err: + table->max_sfs = 0; + kfree(table); + dev->priv.sf_dev_table = NULL; +table_err: + mlx5_core_err(dev, "SF DEV table create err = %d\n", err); +} + +static void mlx5_sf_dev_destroy_all(struct mlx5_sf_dev_table *table) +{ + struct mlx5_sf_dev *sf_dev; + unsigned long index; + + xa_for_each(&table->devices, index, sf_dev) { + xa_erase(&table->devices, index); + mlx5_sf_dev_remove(sf_dev); + } +} + +void mlx5_sf_dev_table_destroy(struct mlx5_core_dev *dev) +{ + struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table; + + if (!table) + return; + + mlx5_vhca_event_notifier_unregister(dev, &table->nb); + + /* Now that event handler is not running, it is safe to destroy + * the sf device without race. + */ + mlx5_sf_dev_destroy_all(table); + + WARN_ON(!xa_empty(&table->devices)); + kfree(table); + dev->priv.sf_dev_table = NULL; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h new file mode 100644 index 000000000000..a6fb7289ba2c --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020 Mellanox Technologies Ltd */ + +#ifndef __MLX5_SF_DEV_H__ +#define __MLX5_SF_DEV_H__ + +#ifdef CONFIG_MLX5_SF + +#include + +#define MLX5_SF_DEV_ID_NAME "sf" + +struct mlx5_sf_dev { + struct auxiliary_device adev; + struct mlx5_core_dev *parent_mdev; + phys_addr_t bar_base_addr; + u32 sfnum; +}; + +void mlx5_sf_dev_table_create(struct mlx5_core_dev *dev); +void mlx5_sf_dev_table_destroy(struct mlx5_core_dev *dev); + +#else + +static inline void mlx5_sf_dev_table_create(struct mlx5_core_dev *dev) +{ +} + +static inline void mlx5_sf_dev_table_destroy(struct mlx5_core_dev *dev) +{ +} + +#endif + +#endif diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index ffba0786051e..08e5fbe97df0 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -508,6 +508,7 @@ struct mlx5_fw_reset; struct mlx5_eq_table; struct mlx5_irq_table; struct mlx5_vhca_state_notifier; +struct mlx5_sf_dev_table; struct mlx5_rate_limit { u32 rate; @@ -606,6 +607,7 @@ struct mlx5_priv { struct mlx5_uars_page *uar; #ifdef CONFIG_MLX5_SF struct mlx5_vhca_state_notifier *vhca_state_notifier; + struct mlx5_sf_dev_table *sf_dev_table; #endif }; -- cgit v1.2.3 From 1958fc2f0712ae771bfd2351b55e5a5b6b0bcfa4 Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Fri, 11 Dec 2020 22:12:18 -0800 Subject: net/mlx5: SF, Add auxiliary device driver Add auxiliary device driver for mlx5 subfunction auxiliary device. A mlx5 subfunction is similar to PCI PF and VF. For a subfunction an auxiliary device is created. As a result, when mlx5 SF auxiliary device binds to the driver, its netdev and rdma device are created, they appear as $ ls -l /sys/bus/auxiliary/devices/ mlx5_core.sf.4 -> ../../../devices/pci0000:00/0000:00:03.0/0000:06:00.0/mlx5_core.sf.4 $ ls -l /sys/class/net/eth1/device /sys/class/net/eth1/device -> ../../../mlx5_core.sf.4 $ cat /sys/bus/auxiliary/devices/mlx5_core.sf.4/sfnum 88 $ devlink dev show pci/0000:06:00.0 auxiliary/mlx5_core.sf.4 $ devlink port show auxiliary/mlx5_core.sf.4/1 auxiliary/mlx5_core.sf.4/1: type eth netdev p0sf88 flavour virtual port 0 splittable false $ rdma link show mlx5_0/1 link mlx5_0/1 state ACTIVE physical_state LINK_UP netdev p0sf88 $ rdma dev show 8: rocep6s0f1: node_type ca fw 16.29.0550 node_guid 248a:0703:00b3:d113 sys_image_guid 248a:0703:00b3:d112 13: mlx5_0: node_type ca fw 16.29.0550 node_guid 0000:00ff:fe00:8888 sys_image_guid 248a:0703:00b3:d112 In future, devlink device instance name will adapt to have sfnum annotation using either an alias or as devlink instance name described in RFC [1]. [1] https://lore.kernel.org/netdev/20200519092258.GF4655@nanopsycho/ Signed-off-by: Parav Pandit Reviewed-by: Vu Pham Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/Makefile | 2 +- drivers/net/ethernet/mellanox/mlx5/core/devlink.c | 12 +++ drivers/net/ethernet/mellanox/mlx5/core/eq.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/main.c | 12 ++- .../net/ethernet/mellanox/mlx5/core/mlx5_core.h | 10 ++ drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c | 20 ++++ .../net/ethernet/mellanox/mlx5/core/sf/dev/dev.c | 10 ++ .../net/ethernet/mellanox/mlx5/core/sf/dev/dev.h | 20 ++++ .../ethernet/mellanox/mlx5/core/sf/dev/driver.c | 101 +++++++++++++++++++++ include/linux/mlx5/driver.h | 4 +- 10 files changed, 187 insertions(+), 6 deletions(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index f5b2e9101348..43fa5efd403c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -89,4 +89,4 @@ mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o # # SF device # -mlx5_core-$(CONFIG_MLX5_SF) += sf/vhca_event.o sf/dev/dev.o +mlx5_core-$(CONFIG_MLX5_SF) += sf/vhca_event.o sf/dev/dev.o sf/dev/driver.o diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index 3261d0dc1104..9afe918c5827 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -7,6 +7,7 @@ #include "fw_reset.h" #include "fs_core.h" #include "eswitch.h" +#include "sf/dev/dev.h" static int mlx5_devlink_flash_update(struct devlink *devlink, struct devlink_flash_update_params *params, @@ -127,6 +128,17 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change, struct netlink_ext_ack *extack) { struct mlx5_core_dev *dev = devlink_priv(devlink); + bool sf_dev_allocated; + + sf_dev_allocated = mlx5_sf_dev_allocated(dev); + if (sf_dev_allocated) { + /* Reload results in deleting SF device which further results in + * unregistering devlink instance while holding devlink_mutext. + * Hence, do not support reload. + */ + NL_SET_ERR_MSG_MOD(extack, "reload is unsupported when SFs are allocated\n"); + return -EOPNOTSUPP; + } switch (action) { case DEVLINK_RELOAD_ACTION_DRIVER_REINIT: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index 421febebc658..174dfbc996c6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -467,7 +467,7 @@ int mlx5_eq_table_init(struct mlx5_core_dev *dev) for (i = 0; i < MLX5_EVENT_TYPE_MAX; i++) ATOMIC_INIT_NOTIFIER_HEAD(&eq_table->nh[i]); - eq_table->irq_table = dev->priv.irq_table; + eq_table->irq_table = mlx5_irq_table_get(dev); return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 26b5502712a6..ab68320e5516 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -84,7 +84,6 @@ unsigned int mlx5_core_debug_mask; module_param_named(debug_mask, mlx5_core_debug_mask, uint, 0644); MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); -#define MLX5_DEFAULT_PROF 2 static unsigned int prof_sel = MLX5_DEFAULT_PROF; module_param_named(prof_sel, prof_sel, uint, 0444); MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); @@ -1303,7 +1302,7 @@ out: mutex_unlock(&dev->intf_state_mutex); } -static int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) +int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) { struct mlx5_priv *priv = &dev->priv; int err; @@ -1353,7 +1352,7 @@ err_health_init: return err; } -static void mlx5_mdev_uninit(struct mlx5_core_dev *dev) +void mlx5_mdev_uninit(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; @@ -1696,6 +1695,10 @@ static int __init init(void) if (err) goto err_debug; + err = mlx5_sf_driver_register(); + if (err) + goto err_sf; + #ifdef CONFIG_MLX5_CORE_EN err = mlx5e_init(); if (err) { @@ -1706,6 +1709,8 @@ static int __init init(void) return 0; +err_sf: + pci_unregister_driver(&mlx5_core_driver); err_debug: mlx5_unregister_debugfs(); return err; @@ -1716,6 +1721,7 @@ static void __exit cleanup(void) #ifdef CONFIG_MLX5_CORE_EN mlx5e_cleanup(); #endif + mlx5_sf_driver_unregister(); pci_unregister_driver(&mlx5_core_driver); mlx5_unregister_debugfs(); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index a33b7496d748..3754ef98554f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -117,6 +117,8 @@ enum mlx5_semaphore_space_address { MLX5_SEMAPHORE_SW_RESET = 0x20, }; +#define MLX5_DEFAULT_PROF 2 + int mlx5_query_hca_caps(struct mlx5_core_dev *dev); int mlx5_query_board_id(struct mlx5_core_dev *dev); int mlx5_cmd_init(struct mlx5_core_dev *dev); @@ -176,6 +178,7 @@ struct cpumask * mlx5_irq_get_affinity_mask(struct mlx5_irq_table *irq_table, int vecidx); struct cpu_rmap *mlx5_irq_get_rmap(struct mlx5_irq_table *table); int mlx5_irq_get_num_comp(struct mlx5_irq_table *table); +struct mlx5_irq_table *mlx5_irq_table_get(struct mlx5_core_dev *dev); int mlx5_events_init(struct mlx5_core_dev *dev); void mlx5_events_cleanup(struct mlx5_core_dev *dev); @@ -257,6 +260,13 @@ enum { u8 mlx5_get_nic_state(struct mlx5_core_dev *dev); void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state); +static inline bool mlx5_core_is_sf(const struct mlx5_core_dev *dev) +{ + return dev->coredev_type == MLX5_COREDEV_SF; +} + +int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx); +void mlx5_mdev_uninit(struct mlx5_core_dev *dev); void mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup); int mlx5_load_one(struct mlx5_core_dev *dev, bool boot); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c index 6fd974920394..a61e09aff152 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c @@ -30,6 +30,9 @@ int mlx5_irq_table_init(struct mlx5_core_dev *dev) { struct mlx5_irq_table *irq_table; + if (mlx5_core_is_sf(dev)) + return 0; + irq_table = kvzalloc(sizeof(*irq_table), GFP_KERNEL); if (!irq_table) return -ENOMEM; @@ -40,6 +43,9 @@ int mlx5_irq_table_init(struct mlx5_core_dev *dev) void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev) { + if (mlx5_core_is_sf(dev)) + return; + kvfree(dev->priv.irq_table); } @@ -268,6 +274,9 @@ int mlx5_irq_table_create(struct mlx5_core_dev *dev) int nvec; int err; + if (mlx5_core_is_sf(dev)) + return 0; + nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + MLX5_IRQ_VEC_COMP_BASE; nvec = min_t(int, nvec, num_eqs); @@ -319,6 +328,9 @@ void mlx5_irq_table_destroy(struct mlx5_core_dev *dev) struct mlx5_irq_table *table = dev->priv.irq_table; int i; + if (mlx5_core_is_sf(dev)) + return; + /* free_irq requires that affinity and rmap will be cleared * before calling it. This is why there is asymmetry with set_rmap * which should be called after alloc_irq but before request_irq. @@ -332,3 +344,11 @@ void mlx5_irq_table_destroy(struct mlx5_core_dev *dev) kfree(table->irq); } +struct mlx5_irq_table *mlx5_irq_table_get(struct mlx5_core_dev *dev) +{ +#ifdef CONFIG_MLX5_SF + if (mlx5_core_is_sf(dev)) + return dev->priv.parent_mdev->priv.irq_table; +#endif + return dev->priv.irq_table; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c index 4a8eeb7c853e..b265f27b2166 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c @@ -24,6 +24,16 @@ static bool mlx5_sf_dev_supported(const struct mlx5_core_dev *dev) return MLX5_CAP_GEN(dev, sf) && mlx5_vhca_event_supported(dev); } +bool mlx5_sf_dev_allocated(const struct mlx5_core_dev *dev) +{ + struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table; + + if (!mlx5_sf_dev_supported(dev)) + return false; + + return !xa_empty(&table->devices); +} + static ssize_t sfnum_show(struct device *dev, struct device_attribute *attr, char *buf) { struct auxiliary_device *adev = container_of(dev, struct auxiliary_device, dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h index a6fb7289ba2c..4de02902aef1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h @@ -13,6 +13,7 @@ struct mlx5_sf_dev { struct auxiliary_device adev; struct mlx5_core_dev *parent_mdev; + struct mlx5_core_dev *mdev; phys_addr_t bar_base_addr; u32 sfnum; }; @@ -20,6 +21,11 @@ struct mlx5_sf_dev { void mlx5_sf_dev_table_create(struct mlx5_core_dev *dev); void mlx5_sf_dev_table_destroy(struct mlx5_core_dev *dev); +int mlx5_sf_driver_register(void); +void mlx5_sf_driver_unregister(void); + +bool mlx5_sf_dev_allocated(const struct mlx5_core_dev *dev); + #else static inline void mlx5_sf_dev_table_create(struct mlx5_core_dev *dev) @@ -30,6 +36,20 @@ static inline void mlx5_sf_dev_table_destroy(struct mlx5_core_dev *dev) { } +static inline int mlx5_sf_driver_register(void) +{ + return 0; +} + +static inline void mlx5_sf_driver_unregister(void) +{ +} + +static inline bool mlx5_sf_dev_allocated(const struct mlx5_core_dev *dev) +{ + return 0; +} + #endif #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c new file mode 100644 index 000000000000..daf63a8115e0 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020 Mellanox Technologies Ltd */ + +#include +#include +#include "mlx5_core.h" +#include "dev.h" +#include "devlink.h" + +static int mlx5_sf_dev_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) +{ + struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev); + struct mlx5_core_dev *mdev; + struct devlink *devlink; + int err; + + devlink = mlx5_devlink_alloc(); + if (!devlink) + return -ENOMEM; + + mdev = devlink_priv(devlink); + mdev->device = &adev->dev; + mdev->pdev = sf_dev->parent_mdev->pdev; + mdev->bar_addr = sf_dev->bar_base_addr; + mdev->iseg_base = sf_dev->bar_base_addr; + mdev->coredev_type = MLX5_COREDEV_SF; + mdev->priv.parent_mdev = sf_dev->parent_mdev; + mdev->priv.adev_idx = adev->id; + sf_dev->mdev = mdev; + + err = mlx5_mdev_init(mdev, MLX5_DEFAULT_PROF); + if (err) { + mlx5_core_warn(mdev, "mlx5_mdev_init on err=%d\n", err); + goto mdev_err; + } + + mdev->iseg = ioremap(mdev->iseg_base, sizeof(*mdev->iseg)); + if (!mdev->iseg) { + mlx5_core_warn(mdev, "remap error\n"); + goto remap_err; + } + + err = mlx5_load_one(mdev, true); + if (err) { + mlx5_core_warn(mdev, "mlx5_load_one err=%d\n", err); + goto load_one_err; + } + return 0; + +load_one_err: + iounmap(mdev->iseg); +remap_err: + mlx5_mdev_uninit(mdev); +mdev_err: + mlx5_devlink_free(devlink); + return err; +} + +static void mlx5_sf_dev_remove(struct auxiliary_device *adev) +{ + struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev); + struct devlink *devlink; + + devlink = priv_to_devlink(sf_dev->mdev); + mlx5_unload_one(sf_dev->mdev, true); + iounmap(sf_dev->mdev->iseg); + mlx5_mdev_uninit(sf_dev->mdev); + mlx5_devlink_free(devlink); +} + +static void mlx5_sf_dev_shutdown(struct auxiliary_device *adev) +{ + struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev); + + mlx5_unload_one(sf_dev->mdev, false); +} + +static const struct auxiliary_device_id mlx5_sf_dev_id_table[] = { + { .name = MLX5_ADEV_NAME "." MLX5_SF_DEV_ID_NAME, }, + { }, +}; + +MODULE_DEVICE_TABLE(auxiliary, mlx5_sf_dev_id_table); + +static struct auxiliary_driver mlx5_sf_driver = { + .name = MLX5_SF_DEV_ID_NAME, + .probe = mlx5_sf_dev_probe, + .remove = mlx5_sf_dev_remove, + .shutdown = mlx5_sf_dev_shutdown, + .id_table = mlx5_sf_dev_id_table, +}; + +int mlx5_sf_driver_register(void) +{ + return auxiliary_driver_register(&mlx5_sf_driver); +} + +void mlx5_sf_driver_unregister(void) +{ + auxiliary_driver_unregister(&mlx5_sf_driver); +} diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 08e5fbe97df0..48e3638b1185 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -193,7 +193,8 @@ enum port_state_policy { enum mlx5_coredev_type { MLX5_COREDEV_PF, - MLX5_COREDEV_VF + MLX5_COREDEV_VF, + MLX5_COREDEV_SF, }; struct mlx5_field_desc { @@ -608,6 +609,7 @@ struct mlx5_priv { #ifdef CONFIG_MLX5_SF struct mlx5_vhca_state_notifier *vhca_state_notifier; struct mlx5_sf_dev_table *sf_dev_table; + struct mlx5_core_dev *parent_mdev; #endif }; -- cgit v1.2.3 From 8f01054186683fe0986d54d584aa13723d51edce Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Fri, 11 Dec 2020 22:12:21 -0800 Subject: net/mlx5: SF, Add port add delete functionality To handle SF port management outside of the eswitch as independent software layer, introduce eswitch notifier APIs so that mlx5 upper layer who wish to support sf port management in switchdev mode can perform its task whenever eswitch mode is set to switchdev or before eswitch is disabled. Initialize sf port table on such eswitch event. Add SF port add and delete functionality in switchdev mode. Destroy all SF ports when eswitch is disabled. Expose SF port add and delete to user via devlink commands. $ devlink dev eswitch set pci/0000:06:00.0 mode switchdev $ devlink port show pci/0000:06:00.0/65535: type eth netdev ens2f0np0 flavour physical port 0 splittable false $ devlink port add pci/0000:06:00.0 flavour pcisf pfnum 0 sfnum 88 pci/0000:06:00.0/32768: type eth netdev eth6 flavour pcisf controller 0 pfnum 0 sfnum 88 external false splittable false function: hw_addr 00:00:00:00:00:00 state inactive opstate detached $ devlink port show ens2f0npf0sf88 pci/0000:06:00.0/32768: type eth netdev ens2f0npf0sf88 flavour pcisf controller 0 pfnum 0 sfnum 88 external false splittable false function: hw_addr 00:00:00:00:00:00 state inactive opstate detached or by its unique port index: $ devlink port show pci/0000:06:00.0/32768 pci/0000:06:00.0/32768: type eth netdev ens2f0npf0sf88 flavour pcisf controller 0 pfnum 0 sfnum 88 external false splittable false function: hw_addr 00:00:00:00:00:00 state inactive opstate detached $ devlink port show ens2f0npf0sf88 -jp { "port": { "pci/0000:06:00.0/32768": { "type": "eth", "netdev": "ens2f0npf0sf88", "flavour": "pcisf", "controller": 0, "pfnum": 0, "sfnum": 88, "external": false, "splittable": false, "function": { "hw_addr": "00:00:00:00:00:00", "state": "inactive", "opstate": "detached" } } } } Signed-off-by: Parav Pandit Reviewed-by: Vu Pham Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/Kconfig | 10 + drivers/net/ethernet/mellanox/mlx5/core/Makefile | 5 + drivers/net/ethernet/mellanox/mlx5/core/cmd.c | 4 + drivers/net/ethernet/mellanox/mlx5/core/devlink.c | 5 + drivers/net/ethernet/mellanox/mlx5/core/eswitch.c | 25 ++ drivers/net/ethernet/mellanox/mlx5/core/eswitch.h | 12 + drivers/net/ethernet/mellanox/mlx5/core/main.c | 18 ++ drivers/net/ethernet/mellanox/mlx5/core/sf/cmd.c | 27 ++ .../net/ethernet/mellanox/mlx5/core/sf/devlink.c | 316 +++++++++++++++++++++ .../net/ethernet/mellanox/mlx5/core/sf/hw_table.c | 125 ++++++++ drivers/net/ethernet/mellanox/mlx5/core/sf/priv.h | 17 ++ drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h | 37 +++ include/linux/mlx5/driver.h | 6 + 13 files changed, 607 insertions(+) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sf/cmd.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sf/priv.h (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig index d6c48582e7a8..ad45d20f9d44 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig +++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig @@ -212,3 +212,13 @@ config MLX5_SF Build support for subfuction device in the NIC. A Mellanox subfunction device can support RDMA, netdevice and vdpa device. It is similar to a SRIOV VF but it doesn't require SRIOV support. + +config MLX5_SF_MANAGER + bool + depends on MLX5_SF && MLX5_ESWITCH + default y + help + Build support for subfuction port in the NIC. A Mellanox subfunction + port is managed through devlink. A subfunction supports RDMA, netdevice + and vdpa device. It is similar to a SRIOV VF but it doesn't require + SRIOV support. diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 43fa5efd403c..40ed31574c80 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -90,3 +90,8 @@ mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o # SF device # mlx5_core-$(CONFIG_MLX5_SF) += sf/vhca_event.o sf/dev/dev.o sf/dev/driver.o + +# +# SF manager +# +mlx5_core-$(CONFIG_MLX5_SF_MANAGER) += sf/cmd.o sf/hw_table.o sf/devlink.o diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index 47dcc3ac2cf0..e8cecd50558d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -333,6 +333,7 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, case MLX5_CMD_OP_DEALLOC_MEMIC: case MLX5_CMD_OP_PAGE_FAULT_RESUME: case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS: + case MLX5_CMD_OP_DEALLOC_SF: return MLX5_CMD_STAT_OK; case MLX5_CMD_OP_QUERY_HCA_CAP: @@ -466,6 +467,7 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, case MLX5_CMD_OP_RELEASE_XRQ_ERROR: case MLX5_CMD_OP_QUERY_VHCA_STATE: case MLX5_CMD_OP_MODIFY_VHCA_STATE: + case MLX5_CMD_OP_ALLOC_SF: *status = MLX5_DRIVER_STATUS_ABORTED; *synd = MLX5_DRIVER_SYND; return -EIO; @@ -661,6 +663,8 @@ const char *mlx5_command_str(int command) MLX5_COMMAND_STR_CASE(MODIFY_XRQ); MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE); MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE); + MLX5_COMMAND_STR_CASE(ALLOC_SF); + MLX5_COMMAND_STR_CASE(DEALLOC_SF); default: return "unknown command opcode"; } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index 9afe918c5827..d4c0cdf5edd9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -8,6 +8,7 @@ #include "fs_core.h" #include "eswitch.h" #include "sf/dev/dev.h" +#include "sf/sf.h" static int mlx5_devlink_flash_update(struct devlink *devlink, struct devlink_flash_update_params *params, @@ -190,6 +191,10 @@ static const struct devlink_ops mlx5_devlink_ops = { .eswitch_encap_mode_get = mlx5_devlink_eswitch_encap_mode_get, .port_function_hw_addr_get = mlx5_devlink_port_function_hw_addr_get, .port_function_hw_addr_set = mlx5_devlink_port_function_hw_addr_set, +#endif +#ifdef CONFIG_MLX5_SF_MANAGER + .port_new = mlx5_devlink_sf_port_new, + .port_del = mlx5_devlink_sf_port_del, #endif .flash_update = mlx5_devlink_flash_update, .info_get = mlx5_devlink_info_get, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 538d2e44a589..820305b1664e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1599,6 +1599,15 @@ mlx5_eswitch_update_num_of_vfs(struct mlx5_eswitch *esw, int num_vfs) kvfree(out); } +static void mlx5_esw_mode_change_notify(struct mlx5_eswitch *esw, u16 mode) +{ + struct mlx5_esw_event_info info = {}; + + info.new_mode = mode; + + blocking_notifier_call_chain(&esw->n_head, 0, &info); +} + /** * mlx5_eswitch_enable_locked - Enable eswitch * @esw: Pointer to eswitch @@ -1659,6 +1668,8 @@ int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs) mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", esw->esw_funcs.num_vfs, esw->enabled_vports); + mlx5_esw_mode_change_notify(esw, mode); + return 0; abort: @@ -1715,6 +1726,11 @@ void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf) esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", esw->esw_funcs.num_vfs, esw->enabled_vports); + /* Notify eswitch users that it is exiting from current mode. + * So that it can do necessary cleanup before the eswitch is disabled. + */ + mlx5_esw_mode_change_notify(esw, MLX5_ESWITCH_NONE); + mlx5_eswitch_event_handlers_unregister(esw); if (esw->mode == MLX5_ESWITCH_LEGACY) @@ -1815,6 +1831,7 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev) esw->offloads.inline_mode = MLX5_INLINE_MODE_NONE; dev->priv.eswitch = esw; + BLOCKING_INIT_NOTIFIER_HEAD(&esw->n_head); return 0; abort: if (esw->work_queue) @@ -2506,4 +2523,12 @@ bool mlx5_esw_multipath_prereq(struct mlx5_core_dev *dev0, dev1->priv.eswitch->mode == MLX5_ESWITCH_OFFLOADS); } +int mlx5_esw_event_notifier_register(struct mlx5_eswitch *esw, struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&esw->n_head, nb); +} +void mlx5_esw_event_notifier_unregister(struct mlx5_eswitch *esw, struct notifier_block *nb) +{ + blocking_notifier_chain_unregister(&esw->n_head, nb); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 54514b04808d..479d2ac2cd85 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -278,6 +278,7 @@ struct mlx5_eswitch { struct { u32 large_group_num; } params; + struct blocking_notifier_head n_head; }; void esw_offloads_disable(struct mlx5_eswitch *esw); @@ -733,6 +734,17 @@ int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_p u16 vport_num, u32 sfnum); void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num); +/** + * mlx5_esw_event_info - Indicates eswitch mode changed/changing. + * + * @new_mode: New mode of eswitch. + */ +struct mlx5_esw_event_info { + u16 new_mode; +}; + +int mlx5_esw_event_notifier_register(struct mlx5_eswitch *esw, struct notifier_block *n); +void mlx5_esw_event_notifier_unregister(struct mlx5_eswitch *esw, struct notifier_block *n); #else /* CONFIG_MLX5_ESWITCH */ /* eswitch API stubs */ static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index ab68320e5516..d6bd09dd7490 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -893,6 +893,18 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) goto err_fpga_cleanup; } + err = mlx5_sf_hw_table_init(dev); + if (err) { + mlx5_core_err(dev, "Failed to init SF HW table %d\n", err); + goto err_sf_hw_table_cleanup; + } + + err = mlx5_sf_table_init(dev); + if (err) { + mlx5_core_err(dev, "Failed to init SF table %d\n", err); + goto err_sf_table_cleanup; + } + dev->dm = mlx5_dm_create(dev); if (IS_ERR(dev->dm)) mlx5_core_warn(dev, "Failed to init device memory%d\n", err); @@ -903,6 +915,10 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) return 0; +err_sf_table_cleanup: + mlx5_sf_hw_table_cleanup(dev); +err_sf_hw_table_cleanup: + mlx5_vhca_event_cleanup(dev); err_fpga_cleanup: mlx5_fpga_cleanup(dev); err_eswitch_cleanup: @@ -936,6 +952,8 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev) mlx5_hv_vhca_destroy(dev->hv_vhca); mlx5_fw_tracer_destroy(dev->tracer); mlx5_dm_cleanup(dev); + mlx5_sf_table_cleanup(dev); + mlx5_sf_hw_table_cleanup(dev); mlx5_vhca_event_cleanup(dev); mlx5_fpga_cleanup(dev); mlx5_eswitch_cleanup(dev->priv.eswitch); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/cmd.c new file mode 100644 index 000000000000..0bc3075f34fa --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/cmd.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020 Mellanox Technologies Ltd */ + +#include +#include "priv.h" + +int mlx5_cmd_alloc_sf(struct mlx5_core_dev *dev, u16 function_id) +{ + u32 out[MLX5_ST_SZ_DW(alloc_sf_out)] = {}; + u32 in[MLX5_ST_SZ_DW(alloc_sf_in)] = {}; + + MLX5_SET(alloc_sf_in, in, opcode, MLX5_CMD_OP_ALLOC_SF); + MLX5_SET(alloc_sf_in, in, function_id, function_id); + + return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} + +int mlx5_cmd_dealloc_sf(struct mlx5_core_dev *dev, u16 function_id) +{ + u32 out[MLX5_ST_SZ_DW(dealloc_sf_out)] = {}; + u32 in[MLX5_ST_SZ_DW(dealloc_sf_in)] = {}; + + MLX5_SET(dealloc_sf_in, in, opcode, MLX5_CMD_OP_DEALLOC_SF); + MLX5_SET(dealloc_sf_in, in, function_id, function_id); + + return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c new file mode 100644 index 000000000000..7ad0d210ec30 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020 Mellanox Technologies Ltd */ + +#include +#include "eswitch.h" +#include "priv.h" + +struct mlx5_sf { + struct devlink_port dl_port; + unsigned int port_index; + u16 id; +}; + +struct mlx5_sf_table { + struct mlx5_core_dev *dev; /* To refer from notifier context. */ + struct xarray port_indices; /* port index based lookup. */ + refcount_t refcount; + struct completion disable_complete; + struct notifier_block esw_nb; +}; + +static struct mlx5_sf * +mlx5_sf_lookup_by_index(struct mlx5_sf_table *table, unsigned int port_index) +{ + return xa_load(&table->port_indices, port_index); +} + +static int mlx5_sf_id_insert(struct mlx5_sf_table *table, struct mlx5_sf *sf) +{ + return xa_insert(&table->port_indices, sf->port_index, sf, GFP_KERNEL); +} + +static void mlx5_sf_id_erase(struct mlx5_sf_table *table, struct mlx5_sf *sf) +{ + xa_erase(&table->port_indices, sf->port_index); +} + +static struct mlx5_sf * +mlx5_sf_alloc(struct mlx5_sf_table *table, u32 sfnum, struct netlink_ext_ack *extack) +{ + unsigned int dl_port_index; + struct mlx5_sf *sf; + u16 hw_fn_id; + int id_err; + int err; + + id_err = mlx5_sf_hw_table_sf_alloc(table->dev, sfnum); + if (id_err < 0) { + err = id_err; + goto id_err; + } + + sf = kzalloc(sizeof(*sf), GFP_KERNEL); + if (!sf) { + err = -ENOMEM; + goto alloc_err; + } + sf->id = id_err; + hw_fn_id = mlx5_sf_sw_to_hw_id(table->dev, sf->id); + dl_port_index = mlx5_esw_vport_to_devlink_port_index(table->dev, hw_fn_id); + sf->port_index = dl_port_index; + + err = mlx5_sf_id_insert(table, sf); + if (err) + goto insert_err; + + return sf; + +insert_err: + kfree(sf); +alloc_err: + mlx5_sf_hw_table_sf_free(table->dev, id_err); +id_err: + if (err == -EEXIST) + NL_SET_ERR_MSG_MOD(extack, "SF already exist. Choose different sfnum"); + return ERR_PTR(err); +} + +static void mlx5_sf_free(struct mlx5_sf_table *table, struct mlx5_sf *sf) +{ + mlx5_sf_id_erase(table, sf); + mlx5_sf_hw_table_sf_free(table->dev, sf->id); + kfree(sf); +} + +static struct mlx5_sf_table *mlx5_sf_table_try_get(struct mlx5_core_dev *dev) +{ + struct mlx5_sf_table *table = dev->priv.sf_table; + + if (!table) + return NULL; + + return refcount_inc_not_zero(&table->refcount) ? table : NULL; +} + +static void mlx5_sf_table_put(struct mlx5_sf_table *table) +{ + if (refcount_dec_and_test(&table->refcount)) + complete(&table->disable_complete); +} + +static int mlx5_sf_add(struct mlx5_core_dev *dev, struct mlx5_sf_table *table, + const struct devlink_port_new_attrs *new_attr, + struct netlink_ext_ack *extack, + unsigned int *new_port_index) +{ + struct mlx5_eswitch *esw = dev->priv.eswitch; + struct mlx5_sf *sf; + u16 hw_fn_id; + int err; + + sf = mlx5_sf_alloc(table, new_attr->sfnum, extack); + if (IS_ERR(sf)) + return PTR_ERR(sf); + + hw_fn_id = mlx5_sf_sw_to_hw_id(dev, sf->id); + err = mlx5_esw_offloads_sf_vport_enable(esw, &sf->dl_port, hw_fn_id, new_attr->sfnum); + if (err) + goto esw_err; + *new_port_index = sf->port_index; + return 0; + +esw_err: + mlx5_sf_free(table, sf); + return err; +} + +static void mlx5_sf_del(struct mlx5_core_dev *dev, struct mlx5_sf_table *table, struct mlx5_sf *sf) +{ + struct mlx5_eswitch *esw = dev->priv.eswitch; + u16 hw_fn_id; + + hw_fn_id = mlx5_sf_sw_to_hw_id(dev, sf->id); + mlx5_esw_offloads_sf_vport_disable(esw, hw_fn_id); + mlx5_sf_free(table, sf); +} + +static int +mlx5_sf_new_check_attr(struct mlx5_core_dev *dev, const struct devlink_port_new_attrs *new_attr, + struct netlink_ext_ack *extack) +{ + if (new_attr->flavour != DEVLINK_PORT_FLAVOUR_PCI_SF) { + NL_SET_ERR_MSG_MOD(extack, "Driver supports only SF port addition"); + return -EOPNOTSUPP; + } + if (new_attr->port_index_valid) { + NL_SET_ERR_MSG_MOD(extack, + "Driver does not support user defined port index assignment"); + return -EOPNOTSUPP; + } + if (!new_attr->sfnum_valid) { + NL_SET_ERR_MSG_MOD(extack, + "User must provide unique sfnum. Driver does not support auto assignment"); + return -EOPNOTSUPP; + } + if (new_attr->controller_valid && new_attr->controller) { + NL_SET_ERR_MSG_MOD(extack, "External controller is unsupported"); + return -EOPNOTSUPP; + } + if (new_attr->pfnum != PCI_FUNC(dev->pdev->devfn)) { + NL_SET_ERR_MSG_MOD(extack, "Invalid pfnum supplied"); + return -EOPNOTSUPP; + } + return 0; +} + +int mlx5_devlink_sf_port_new(struct devlink *devlink, + const struct devlink_port_new_attrs *new_attr, + struct netlink_ext_ack *extack, + unsigned int *new_port_index) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + struct mlx5_sf_table *table; + int err; + + err = mlx5_sf_new_check_attr(dev, new_attr, extack); + if (err) + return err; + + table = mlx5_sf_table_try_get(dev); + if (!table) { + NL_SET_ERR_MSG_MOD(extack, + "Port add is only supported in eswitch switchdev mode or SF ports are disabled."); + return -EOPNOTSUPP; + } + err = mlx5_sf_add(dev, table, new_attr, extack, new_port_index); + mlx5_sf_table_put(table); + return err; +} + +int mlx5_devlink_sf_port_del(struct devlink *devlink, unsigned int port_index, + struct netlink_ext_ack *extack) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + struct mlx5_sf_table *table; + struct mlx5_sf *sf; + int err = 0; + + table = mlx5_sf_table_try_get(dev); + if (!table) { + NL_SET_ERR_MSG_MOD(extack, + "Port del is only supported in eswitch switchdev mode or SF ports are disabled."); + return -EOPNOTSUPP; + } + sf = mlx5_sf_lookup_by_index(table, port_index); + if (!sf) { + err = -ENODEV; + goto sf_err; + } + + mlx5_sf_del(dev, table, sf); +sf_err: + mlx5_sf_table_put(table); + return err; +} + +static void mlx5_sf_destroy_all(struct mlx5_sf_table *table) +{ + struct mlx5_core_dev *dev = table->dev; + unsigned long index; + struct mlx5_sf *sf; + + xa_for_each(&table->port_indices, index, sf) + mlx5_sf_del(dev, table, sf); +} + +static void mlx5_sf_table_enable(struct mlx5_sf_table *table) +{ + if (!mlx5_sf_max_functions(table->dev)) + return; + + init_completion(&table->disable_complete); + refcount_set(&table->refcount, 1); +} + +static void mlx5_sf_table_disable(struct mlx5_sf_table *table) +{ + if (!mlx5_sf_max_functions(table->dev)) + return; + + if (!refcount_read(&table->refcount)) + return; + + /* Balances with refcount_set; drop the reference so that new user cmd cannot start. */ + mlx5_sf_table_put(table); + wait_for_completion(&table->disable_complete); + + /* At this point, no new user commands can start. + * It is safe to destroy all user created SFs. + */ + mlx5_sf_destroy_all(table); +} + +static int mlx5_sf_esw_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct mlx5_sf_table *table = container_of(nb, struct mlx5_sf_table, esw_nb); + const struct mlx5_esw_event_info *mode = data; + + switch (mode->new_mode) { + case MLX5_ESWITCH_OFFLOADS: + mlx5_sf_table_enable(table); + break; + case MLX5_ESWITCH_NONE: + mlx5_sf_table_disable(table); + break; + default: + break; + }; + + return 0; +} + +static bool mlx5_sf_table_supported(const struct mlx5_core_dev *dev) +{ + return dev->priv.eswitch && MLX5_ESWITCH_MANAGER(dev) && mlx5_sf_supported(dev); +} + +int mlx5_sf_table_init(struct mlx5_core_dev *dev) +{ + struct mlx5_sf_table *table; + int err; + + if (!mlx5_sf_table_supported(dev)) + return 0; + + table = kzalloc(sizeof(*table), GFP_KERNEL); + if (!table) + return -ENOMEM; + + table->dev = dev; + xa_init(&table->port_indices); + dev->priv.sf_table = table; + table->esw_nb.notifier_call = mlx5_sf_esw_event; + err = mlx5_esw_event_notifier_register(dev->priv.eswitch, &table->esw_nb); + if (err) + goto reg_err; + return 0; + +reg_err: + kfree(table); + dev->priv.sf_table = NULL; + return err; +} + +void mlx5_sf_table_cleanup(struct mlx5_core_dev *dev) +{ + struct mlx5_sf_table *table = dev->priv.sf_table; + + if (!table) + return; + + mlx5_esw_event_notifier_unregister(dev->priv.eswitch, &table->esw_nb); + WARN_ON(refcount_read(&table->refcount)); + WARN_ON(!xa_empty(&table->port_indices)); + kfree(table); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c new file mode 100644 index 000000000000..c7757f399e8a --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020 Mellanox Technologies Ltd */ +#include +#include "vhca_event.h" +#include "priv.h" +#include "sf.h" +#include "ecpf.h" + +struct mlx5_sf_hw { + u32 usr_sfnum; + u8 allocated: 1; +}; + +struct mlx5_sf_hw_table { + struct mlx5_core_dev *dev; + struct mlx5_sf_hw *sfs; + int max_local_functions; + u8 ecpu: 1; +}; + +u16 mlx5_sf_sw_to_hw_id(const struct mlx5_core_dev *dev, u16 sw_id) +{ + return sw_id + mlx5_sf_start_function_id(dev); +} + +int mlx5_sf_hw_table_sf_alloc(struct mlx5_core_dev *dev, u32 usr_sfnum) +{ + struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table; + int sw_id = -ENOSPC; + u16 hw_fn_id; + int err; + int i; + + if (!table->max_local_functions) + return -EOPNOTSUPP; + + /* Check if sf with same sfnum already exists or not. */ + for (i = 0; i < table->max_local_functions; i++) { + if (table->sfs[i].allocated && table->sfs[i].usr_sfnum == usr_sfnum) + return -EEXIST; + } + + /* Find the free entry and allocate the entry from the array */ + for (i = 0; i < table->max_local_functions; i++) { + if (!table->sfs[i].allocated) { + table->sfs[i].usr_sfnum = usr_sfnum; + table->sfs[i].allocated = true; + sw_id = i; + break; + } + } + if (sw_id == -ENOSPC) { + err = -ENOSPC; + goto err; + } + + hw_fn_id = mlx5_sf_sw_to_hw_id(table->dev, sw_id); + err = mlx5_cmd_alloc_sf(table->dev, hw_fn_id); + if (err) + goto err; + + err = mlx5_modify_vhca_sw_id(dev, hw_fn_id, table->ecpu, usr_sfnum); + if (err) + goto vhca_err; + + return sw_id; + +vhca_err: + mlx5_cmd_dealloc_sf(table->dev, hw_fn_id); +err: + table->sfs[i].allocated = false; + return err; +} + +void mlx5_sf_hw_table_sf_free(struct mlx5_core_dev *dev, u16 id) +{ + struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table; + u16 hw_fn_id; + + hw_fn_id = mlx5_sf_sw_to_hw_id(table->dev, id); + mlx5_cmd_dealloc_sf(table->dev, hw_fn_id); + table->sfs[id].allocated = false; +} + +int mlx5_sf_hw_table_init(struct mlx5_core_dev *dev) +{ + struct mlx5_sf_hw_table *table; + struct mlx5_sf_hw *sfs; + int max_functions; + + if (!mlx5_sf_supported(dev)) + return 0; + + max_functions = mlx5_sf_max_functions(dev); + table = kzalloc(sizeof(*table), GFP_KERNEL); + if (!table) + return -ENOMEM; + + sfs = kcalloc(max_functions, sizeof(*sfs), GFP_KERNEL); + if (!sfs) + goto table_err; + + table->dev = dev; + table->sfs = sfs; + table->max_local_functions = max_functions; + table->ecpu = mlx5_read_embedded_cpu(dev); + dev->priv.sf_hw_table = table; + mlx5_core_dbg(dev, "SF HW table: max sfs = %d\n", max_functions); + return 0; + +table_err: + kfree(table); + return -ENOMEM; +} + +void mlx5_sf_hw_table_cleanup(struct mlx5_core_dev *dev) +{ + struct mlx5_sf_hw_table *table = dev->priv.sf_hw_table; + + if (!table) + return; + + kfree(table->sfs); + kfree(table); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/priv.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/priv.h new file mode 100644 index 000000000000..7f3622375a9c --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/priv.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020 Mellanox Technologies Ltd */ + +#ifndef __MLX5_SF_PRIV_H__ +#define __MLX5_SF_PRIV_H__ + +#include + +int mlx5_cmd_alloc_sf(struct mlx5_core_dev *dev, u16 function_id); +int mlx5_cmd_dealloc_sf(struct mlx5_core_dev *dev, u16 function_id); + +u16 mlx5_sf_sw_to_hw_id(const struct mlx5_core_dev *dev, u16 sw_id); + +int mlx5_sf_hw_table_sf_alloc(struct mlx5_core_dev *dev, u32 usr_sfnum); +void mlx5_sf_hw_table_sf_free(struct mlx5_core_dev *dev, u16 id); + +#endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h index 623191679b49..31278dc42e72 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h @@ -42,4 +42,41 @@ static inline u16 mlx5_sf_max_functions(const struct mlx5_core_dev *dev) #endif +#ifdef CONFIG_MLX5_SF_MANAGER + +int mlx5_sf_hw_table_init(struct mlx5_core_dev *dev); +void mlx5_sf_hw_table_cleanup(struct mlx5_core_dev *dev); + +int mlx5_sf_table_init(struct mlx5_core_dev *dev); +void mlx5_sf_table_cleanup(struct mlx5_core_dev *dev); + +int mlx5_devlink_sf_port_new(struct devlink *devlink, + const struct devlink_port_new_attrs *add_attr, + struct netlink_ext_ack *extack, + unsigned int *new_port_index); +int mlx5_devlink_sf_port_del(struct devlink *devlink, unsigned int port_index, + struct netlink_ext_ack *extack); + +#else + +static inline int mlx5_sf_hw_table_init(struct mlx5_core_dev *dev) +{ + return 0; +} + +static inline void mlx5_sf_hw_table_cleanup(struct mlx5_core_dev *dev) +{ +} + +static inline int mlx5_sf_table_init(struct mlx5_core_dev *dev) +{ + return 0; +} + +static inline void mlx5_sf_table_cleanup(struct mlx5_core_dev *dev) +{ +} + +#endif + #endif diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 48e3638b1185..7e357c7f0d5e 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -510,6 +510,8 @@ struct mlx5_eq_table; struct mlx5_irq_table; struct mlx5_vhca_state_notifier; struct mlx5_sf_dev_table; +struct mlx5_sf_hw_table; +struct mlx5_sf_table; struct mlx5_rate_limit { u32 rate; @@ -611,6 +613,10 @@ struct mlx5_priv { struct mlx5_sf_dev_table *sf_dev_table; struct mlx5_core_dev *parent_mdev; #endif +#ifdef CONFIG_MLX5_SF_MANAGER + struct mlx5_sf_hw_table *sf_hw_table; + struct mlx5_sf_table *sf_table; +#endif }; enum mlx5_device_state { -- cgit v1.2.3 From 8e7f37f2aaa56b723a24f6872817cf9c6410b613 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 7 Dec 2020 17:41:02 -0800 Subject: mm: Add mem_dump_obj() to print source of memory block There are kernel facilities such as per-CPU reference counts that give error messages in generic handlers or callbacks, whose messages are unenlightening. In the case of per-CPU reference-count underflow, this is not a problem when creating a new use of this facility because in that case the bug is almost certainly in the code implementing that new use. However, trouble arises when deploying across many systems, which might exercise corner cases that were not seen during development and testing. Here, it would be really nice to get some kind of hint as to which of several uses the underflow was caused by. This commit therefore exposes a mem_dump_obj() function that takes a pointer to memory (which must still be allocated if it has been dynamically allocated) and prints available information on where that memory came from. This pointer can reference the middle of the block as well as the beginning of the block, as needed by things like RCU callback functions and timer handlers that might not know where the beginning of the memory block is. These functions and handlers can use mem_dump_obj() to print out better hints as to where the problem might lie. The information printed can depend on kernel configuration. For example, the allocation return address can be printed only for slab and slub, and even then only when the necessary debug has been enabled. For slab, build with CONFIG_DEBUG_SLAB=y, and either use sizes with ample space to the next power of two or use the SLAB_STORE_USER when creating the kmem_cache structure. For slub, build with CONFIG_SLUB_DEBUG=y and boot with slub_debug=U, or pass SLAB_STORE_USER to kmem_cache_create() if more focused use is desired. Also for slub, use CONFIG_STACKTRACE to enable printing of the allocation-time stack trace. Cc: Christoph Lameter Cc: Pekka Enberg Cc: David Rientjes Cc: Joonsoo Kim Cc: Andrew Morton Cc: Reported-by: Andrii Nakryiko [ paulmck: Convert to printing and change names per Joonsoo Kim. ] [ paulmck: Move slab definition per Stephen Rothwell and kbuild test robot. ] [ paulmck: Handle CONFIG_MMU=n case where vmalloc() is kmalloc(). ] [ paulmck: Apply Vlastimil Babka feedback on slab.c kmem_provenance(). ] [ paulmck: Extract more info from !SLUB_DEBUG per Joonsoo Kim. ] [ paulmck: Explicitly check for small pointers per Naresh Kamboju. ] Acked-by: Joonsoo Kim Acked-by: Vlastimil Babka Tested-by: Naresh Kamboju Signed-off-by: Paul E. McKenney --- include/linux/mm.h | 2 ++ include/linux/slab.h | 2 ++ mm/slab.c | 20 ++++++++++++++ mm/slab.h | 12 +++++++++ mm/slab_common.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++ mm/slob.c | 6 +++++ mm/slub.c | 40 ++++++++++++++++++++++++++++ mm/util.c | 24 +++++++++++++++++ 8 files changed, 181 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 5299b90a6c40..af7d050900e7 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3169,5 +3169,7 @@ unsigned long wp_shared_mapping_range(struct address_space *mapping, extern int sysctl_nr_trim_pages; +void mem_dump_obj(void *object); + #endif /* __KERNEL__ */ #endif /* _LINUX_MM_H */ diff --git a/include/linux/slab.h b/include/linux/slab.h index be4ba5867ac5..7ae604076767 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -186,6 +186,8 @@ void kfree(const void *); void kfree_sensitive(const void *); size_t __ksize(const void *); size_t ksize(const void *); +bool kmem_valid_obj(void *object); +void kmem_dump_obj(void *object); #ifdef CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR void __check_heap_object(const void *ptr, unsigned long n, struct page *page, diff --git a/mm/slab.c b/mm/slab.c index d7c8da9319c7..dcc55e78f353 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3635,6 +3635,26 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t flags, EXPORT_SYMBOL(__kmalloc_node_track_caller); #endif /* CONFIG_NUMA */ +void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page) +{ + struct kmem_cache *cachep; + unsigned int objnr; + void *objp; + + kpp->kp_ptr = object; + kpp->kp_page = page; + cachep = page->slab_cache; + kpp->kp_slab_cache = cachep; + objp = object - obj_offset(cachep); + kpp->kp_data_offset = obj_offset(cachep); + page = virt_to_head_page(objp); + objnr = obj_to_index(cachep, page, objp); + objp = index_to_obj(cachep, page, objnr); + kpp->kp_objp = objp; + if (DEBUG && cachep->flags & SLAB_STORE_USER) + kpp->kp_ret = *dbg_userword(cachep, objp); +} + /** * __do_kmalloc - allocate memory * @size: how many bytes of memory are required. diff --git a/mm/slab.h b/mm/slab.h index 1a756a359fa8..ecad9b57bc44 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -615,4 +615,16 @@ static inline bool slab_want_init_on_free(struct kmem_cache *c) return false; } +#define KS_ADDRS_COUNT 16 +struct kmem_obj_info { + void *kp_ptr; + struct page *kp_page; + void *kp_objp; + unsigned long kp_data_offset; + struct kmem_cache *kp_slab_cache; + void *kp_ret; + void *kp_stack[KS_ADDRS_COUNT]; +}; +void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page); + #endif /* MM_SLAB_H */ diff --git a/mm/slab_common.c b/mm/slab_common.c index e981c80d216c..adbace4256ef 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -537,6 +537,81 @@ bool slab_is_available(void) return slab_state >= UP; } +/** + * kmem_valid_obj - does the pointer reference a valid slab object? + * @object: pointer to query. + * + * Return: %true if the pointer is to a not-yet-freed object from + * kmalloc() or kmem_cache_alloc(), either %true or %false if the pointer + * is to an already-freed object, and %false otherwise. + */ +bool kmem_valid_obj(void *object) +{ + struct page *page; + + /* Some arches consider ZERO_SIZE_PTR to be a valid address. */ + if (object < (void *)PAGE_SIZE || !virt_addr_valid(object)) + return false; + page = virt_to_head_page(object); + return PageSlab(page); +} + +/** + * kmem_dump_obj - Print available slab provenance information + * @object: slab object for which to find provenance information. + * + * This function uses pr_cont(), so that the caller is expected to have + * printed out whatever preamble is appropriate. The provenance information + * depends on the type of object and on how much debugging is enabled. + * For a slab-cache object, the fact that it is a slab object is printed, + * and, if available, the slab name, return address, and stack trace from + * the allocation of that object. + * + * This function will splat if passed a pointer to a non-slab object. + * If you are not sure what type of object you have, you should instead + * use mem_dump_obj(). + */ +void kmem_dump_obj(void *object) +{ + char *cp = IS_ENABLED(CONFIG_MMU) ? "" : "/vmalloc"; + int i; + struct page *page; + unsigned long ptroffset; + struct kmem_obj_info kp = { }; + + if (WARN_ON_ONCE(!virt_addr_valid(object))) + return; + page = virt_to_head_page(object); + if (WARN_ON_ONCE(!PageSlab(page))) { + pr_cont(" non-slab memory.\n"); + return; + } + kmem_obj_info(&kp, object, page); + if (kp.kp_slab_cache) + pr_cont(" slab%s %s", cp, kp.kp_slab_cache->name); + else + pr_cont(" slab%s", cp); + if (kp.kp_objp) + pr_cont(" start %px", kp.kp_objp); + if (kp.kp_data_offset) + pr_cont(" data offset %lu", kp.kp_data_offset); + if (kp.kp_objp) { + ptroffset = ((char *)object - (char *)kp.kp_objp) - kp.kp_data_offset; + pr_cont(" pointer offset %lu", ptroffset); + } + if (kp.kp_slab_cache && kp.kp_slab_cache->usersize) + pr_cont(" size %u", kp.kp_slab_cache->usersize); + if (kp.kp_ret) + pr_cont(" allocated at %pS\n", kp.kp_ret); + else + pr_cont("\n"); + for (i = 0; i < ARRAY_SIZE(kp.kp_stack); i++) { + if (!kp.kp_stack[i]) + break; + pr_info(" %pS\n", kp.kp_stack[i]); + } +} + #ifndef CONFIG_SLOB /* Create a cache during boot when no slab services are available yet */ void __init create_boot_cache(struct kmem_cache *s, const char *name, diff --git a/mm/slob.c b/mm/slob.c index 8d4bfa46247f..ef87ada8705d 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -461,6 +461,12 @@ out: spin_unlock_irqrestore(&slob_lock, flags); } +void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page) +{ + kpp->kp_ptr = object; + kpp->kp_page = page; +} + /* * End of slob allocator proper. Begin kmem_cache_alloc and kmalloc frontend. */ diff --git a/mm/slub.c b/mm/slub.c index 0c8b43a5b3b0..3c1a84316fd7 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -3919,6 +3919,46 @@ int __kmem_cache_shutdown(struct kmem_cache *s) return 0; } +void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct page *page) +{ + void *base; + int __maybe_unused i; + unsigned int objnr; + void *objp; + void *objp0; + struct kmem_cache *s = page->slab_cache; + struct track __maybe_unused *trackp; + + kpp->kp_ptr = object; + kpp->kp_page = page; + kpp->kp_slab_cache = s; + base = page_address(page); + objp0 = kasan_reset_tag(object); +#ifdef CONFIG_SLUB_DEBUG + objp = restore_red_left(s, objp0); +#else + objp = objp0; +#endif + objnr = obj_to_index(s, page, objp); + kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp); + objp = base + s->size * objnr; + kpp->kp_objp = objp; + if (WARN_ON_ONCE(objp < base || objp >= base + page->objects * s->size || (objp - base) % s->size) || + !(s->flags & SLAB_STORE_USER)) + return; +#ifdef CONFIG_SLUB_DEBUG + trackp = get_track(s, objp, TRACK_ALLOC); + kpp->kp_ret = (void *)trackp->addr; +#ifdef CONFIG_STACKTRACE + for (i = 0; i < KS_ADDRS_COUNT && i < TRACK_ADDRS_COUNT; i++) { + kpp->kp_stack[i] = (void *)trackp->addrs[i]; + if (!kpp->kp_stack[i]) + break; + } +#endif +#endif +} + /******************************************************************** * Kmalloc subsystem *******************************************************************/ diff --git a/mm/util.c b/mm/util.c index 8c9b7d1e7c49..da46f9d6c382 100644 --- a/mm/util.c +++ b/mm/util.c @@ -982,3 +982,27 @@ int __weak memcmp_pages(struct page *page1, struct page *page2) kunmap_atomic(addr1); return ret; } + +/** + * mem_dump_obj - Print available provenance information + * @object: object for which to find provenance information. + * + * This function uses pr_cont(), so that the caller is expected to have + * printed out whatever preamble is appropriate. The provenance information + * depends on the type of object and on how much debugging is enabled. + * For example, for a slab-cache object, the slab name is printed, and, + * if available, the return address and stack trace from the allocation + * of that object. + */ +void mem_dump_obj(void *object) +{ + if (!virt_addr_valid(object)) { + pr_cont(" non-paged (local) memory.\n"); + return; + } + if (kmem_valid_obj(object)) { + kmem_dump_obj(object); + return; + } + pr_cont(" non-slab memory.\n"); +} -- cgit v1.2.3 From 98f180837a896ecedf8f7e12af22b57f271d43c9 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 8 Dec 2020 16:13:57 -0800 Subject: mm: Make mem_dump_obj() handle vmalloc() memory This commit adds vmalloc() support to mem_dump_obj(). Note that the vmalloc_dump_obj() function combines the checking and dumping, in contrast with the split between kmem_valid_obj() and kmem_dump_obj(). The reason for the difference is that the checking in the vmalloc() case involves acquiring a global lock, and redundant acquisitions of global locks should be avoided, even on not-so-fast paths. Note that this change causes on-stack variables to be reported as vmalloc() storage from kernel_clone() or similar, depending on the degree of inlining that your compiler does. This is likely more helpful than the earlier "non-paged (local) memory". Cc: Andrew Morton Cc: Joonsoo Kim Cc: Reported-by: Andrii Nakryiko Acked-by: Vlastimil Babka Tested-by: Naresh Kamboju Signed-off-by: Paul E. McKenney --- include/linux/vmalloc.h | 6 ++++++ mm/util.c | 14 ++++++++------ mm/vmalloc.c | 12 ++++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 80c0181c411d..c18f4751a704 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -246,4 +246,10 @@ pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) int register_vmap_purge_notifier(struct notifier_block *nb); int unregister_vmap_purge_notifier(struct notifier_block *nb); +#ifdef CONFIG_MMU +bool vmalloc_dump_obj(void *object); +#else +static inline bool vmalloc_dump_obj(void *object) { return false; } +#endif + #endif /* _LINUX_VMALLOC_H */ diff --git a/mm/util.c b/mm/util.c index 92f23d2c1277..54870226cea6 100644 --- a/mm/util.c +++ b/mm/util.c @@ -996,18 +996,20 @@ int __weak memcmp_pages(struct page *page1, struct page *page2) */ void mem_dump_obj(void *object) { + if (kmem_valid_obj(object)) { + kmem_dump_obj(object); + return; + } + if (vmalloc_dump_obj(object)) + return; if (!virt_addr_valid(object)) { if (object == NULL) pr_cont(" NULL pointer.\n"); else if (object == ZERO_SIZE_PTR) pr_cont(" zero-size pointer.\n"); else - pr_cont(" non-paged (local) memory.\n"); - return; - } - if (kmem_valid_obj(object)) { - kmem_dump_obj(object); + pr_cont(" non-paged memory.\n"); return; } - pr_cont(" non-slab memory.\n"); + pr_cont(" non-slab/vmalloc memory.\n"); } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 4d88fe5a277a..c274ea4f14ea 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -3448,6 +3448,18 @@ void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) } #endif /* CONFIG_SMP */ +bool vmalloc_dump_obj(void *object) +{ + struct vm_struct *vm; + void *objp = (void *)PAGE_ALIGN((unsigned long)object); + + vm = find_vm_area(objp); + if (!vm) + return false; + pr_cont(" vmalloc allocated at %pS\n", vm->caller); + return true; +} + #ifdef CONFIG_PROC_FS static void *s_start(struct seq_file *m, loff_t *pos) __acquires(&vmap_purge_lock) -- cgit v1.2.3 From e7ed11ee945438b737e2ae2370e35591e16ec371 Mon Sep 17 00:00:00 2001 From: Yousuk Seung Date: Wed, 20 Jan 2021 12:41:55 -0800 Subject: tcp: add TTL to SCM_TIMESTAMPING_OPT_STATS This patch adds TCP_NLA_TTL to SCM_TIMESTAMPING_OPT_STATS that exports the time-to-live or hop limit of the latest incoming packet with SCM_TSTAMP_ACK. The value exported may not be from the packet that acks the sequence when incoming packets are aggregated. Exporting the time-to-live or hop limit value of incoming packets helps to estimate the hop count of the path of the flow that may change over time. Signed-off-by: Yousuk Seung Signed-off-by: Eric Dumazet Signed-off-by: Neal Cardwell Link: https://lore.kernel.org/r/20210120204155.552275-1-ysseung@google.com Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 2 +- include/linux/tcp.h | 3 ++- include/uapi/linux/tcp.h | 1 + net/core/dev.c | 2 +- net/core/skbuff.c | 6 ++++-- net/ipv4/tcp.c | 18 +++++++++++++++++- net/ipv4/tcp_input.c | 16 ++++++++-------- 7 files changed, 34 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 186dad231e30..9313b5aaf45b 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3859,7 +3859,7 @@ static inline bool skb_defer_rx_timestamp(struct sk_buff *skb) void skb_complete_tx_timestamp(struct sk_buff *skb, struct skb_shared_hwtstamps *hwtstamps); -void __skb_tstamp_tx(struct sk_buff *orig_skb, +void __skb_tstamp_tx(struct sk_buff *orig_skb, const struct sk_buff *ack_skb, struct skb_shared_hwtstamps *hwtstamps, struct sock *sk, int tstype); diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 2f87377e9af7..48d8a363319e 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -496,7 +496,8 @@ static inline u32 tcp_saved_syn_len(const struct saved_syn *saved_syn) } struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk, - const struct sk_buff *orig_skb); + const struct sk_buff *orig_skb, + const struct sk_buff *ack_skb); static inline u16 tcp_mss_clamp(const struct tcp_sock *tp, u16 mss) { diff --git a/include/uapi/linux/tcp.h b/include/uapi/linux/tcp.h index 768e93bd5b51..16dfa40bdac3 100644 --- a/include/uapi/linux/tcp.h +++ b/include/uapi/linux/tcp.h @@ -314,6 +314,7 @@ enum { TCP_NLA_TIMEOUT_REHASH, /* Timeout-triggered rehash attempts */ TCP_NLA_BYTES_NOTSENT, /* Bytes in write queue not yet sent */ TCP_NLA_EDT, /* Earliest departure time (CLOCK_MONOTONIC) */ + TCP_NLA_TTL, /* TTL or hop limit of a packet received */ }; /* for TCP_MD5SIG socket option */ diff --git a/net/core/dev.c b/net/core/dev.c index d9ce02e95992..6df3f1bcdc68 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4084,7 +4084,7 @@ static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) skb_reset_mac_header(skb); if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) - __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED); + __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED); /* Disable soft irqs for various locks below. Also * stops preemption for RCU. diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 145503d3f06b..2af12f7e170c 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -4721,6 +4721,7 @@ err: EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); void __skb_tstamp_tx(struct sk_buff *orig_skb, + const struct sk_buff *ack_skb, struct skb_shared_hwtstamps *hwtstamps, struct sock *sk, int tstype) { @@ -4743,7 +4744,8 @@ void __skb_tstamp_tx(struct sk_buff *orig_skb, if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) && sk->sk_protocol == IPPROTO_TCP && sk->sk_type == SOCK_STREAM) { - skb = tcp_get_timestamping_opt_stats(sk, orig_skb); + skb = tcp_get_timestamping_opt_stats(sk, orig_skb, + ack_skb); opt_stats = true; } else #endif @@ -4772,7 +4774,7 @@ EXPORT_SYMBOL_GPL(__skb_tstamp_tx); void skb_tstamp_tx(struct sk_buff *orig_skb, struct skb_shared_hwtstamps *hwtstamps) { - return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk, + return __skb_tstamp_tx(orig_skb, NULL, hwtstamps, orig_skb->sk, SCM_TSTAMP_SND); } EXPORT_SYMBOL_GPL(skb_tstamp_tx); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 856ae516ac18..a1a17b64f8cd 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -3767,11 +3767,24 @@ static size_t tcp_opt_stats_get_size(void) nla_total_size(sizeof(u16)) + /* TCP_NLA_TIMEOUT_REHASH */ nla_total_size(sizeof(u32)) + /* TCP_NLA_BYTES_NOTSENT */ nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_EDT */ + nla_total_size(sizeof(u8)) + /* TCP_NLA_TTL */ 0; } +/* Returns TTL or hop limit of an incoming packet from skb. */ +static u8 tcp_skb_ttl_or_hop_limit(const struct sk_buff *skb) +{ + if (skb->protocol == htons(ETH_P_IP)) + return ip_hdr(skb)->ttl; + else if (skb->protocol == htons(ETH_P_IPV6)) + return ipv6_hdr(skb)->hop_limit; + else + return 0; +} + struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk, - const struct sk_buff *orig_skb) + const struct sk_buff *orig_skb, + const struct sk_buff *ack_skb) { const struct tcp_sock *tp = tcp_sk(sk); struct sk_buff *stats; @@ -3827,6 +3840,9 @@ struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk, max_t(int, 0, tp->write_seq - tp->snd_nxt)); nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns, TCP_NLA_PAD); + if (ack_skb) + nla_put_u8(stats, TCP_NLA_TTL, + tcp_skb_ttl_or_hop_limit(ack_skb)); return stats; } diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index a7dfca0a38cd..d4f66aba9fd8 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -3145,7 +3145,7 @@ static u32 tcp_tso_acked(struct sock *sk, struct sk_buff *skb) } static void tcp_ack_tstamp(struct sock *sk, struct sk_buff *skb, - u32 prior_snd_una) + const struct sk_buff *ack_skb, u32 prior_snd_una) { const struct skb_shared_info *shinfo; @@ -3157,7 +3157,7 @@ static void tcp_ack_tstamp(struct sock *sk, struct sk_buff *skb, if (!before(shinfo->tskey, prior_snd_una) && before(shinfo->tskey, tcp_sk(sk)->snd_una)) { tcp_skb_tsorted_save(skb) { - __skb_tstamp_tx(skb, NULL, sk, SCM_TSTAMP_ACK); + __skb_tstamp_tx(skb, ack_skb, NULL, sk, SCM_TSTAMP_ACK); } tcp_skb_tsorted_restore(skb); } } @@ -3166,8 +3166,8 @@ static void tcp_ack_tstamp(struct sock *sk, struct sk_buff *skb, * is before the ack sequence we can discard it as it's confirmed to have * arrived at the other end. */ -static int tcp_clean_rtx_queue(struct sock *sk, u32 prior_fack, - u32 prior_snd_una, +static int tcp_clean_rtx_queue(struct sock *sk, const struct sk_buff *ack_skb, + u32 prior_fack, u32 prior_snd_una, struct tcp_sacktag_state *sack, bool ece_ack) { const struct inet_connection_sock *icsk = inet_csk(sk); @@ -3256,7 +3256,7 @@ static int tcp_clean_rtx_queue(struct sock *sk, u32 prior_fack, if (!fully_acked) break; - tcp_ack_tstamp(sk, skb, prior_snd_una); + tcp_ack_tstamp(sk, skb, ack_skb, prior_snd_una); next = skb_rb_next(skb); if (unlikely(skb == tp->retransmit_skb_hint)) @@ -3274,7 +3274,7 @@ static int tcp_clean_rtx_queue(struct sock *sk, u32 prior_fack, tp->snd_up = tp->snd_una; if (skb) { - tcp_ack_tstamp(sk, skb, prior_snd_una); + tcp_ack_tstamp(sk, skb, ack_skb, prior_snd_una); if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED) flag |= FLAG_SACK_RENEGING; } @@ -3809,8 +3809,8 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) goto no_queue; /* See if we can take anything off of the retransmit queue. */ - flag |= tcp_clean_rtx_queue(sk, prior_fack, prior_snd_una, &sack_state, - flag & FLAG_ECE); + flag |= tcp_clean_rtx_queue(sk, skb, prior_fack, prior_snd_una, + &sack_state, flag & FLAG_ECE); tcp_rack_update_reo_wnd(sk, &rs); -- cgit v1.2.3 From d03b195b5aa015f6c11988b86a3625f8d5dbac52 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Tue, 19 Jan 2021 14:08:13 +0200 Subject: sch_htb: Hierarchical QoS hardware offload HTB doesn't scale well because of contention on a single lock, and it also consumes CPU. This patch adds support for offloading HTB to hardware that supports hierarchical rate limiting. In the offload mode, HTB passes control commands to the driver using ndo_setup_tc. The driver has to replicate the whole hierarchy of classes and their settings (rate, ceil) in the NIC. Every modification of the HTB tree caused by the admin results in ndo_setup_tc being called. After this setup, the HTB algorithm is done completely in the NIC. An SQ (send queue) is created for every leaf class and attached to the hierarchy, so that the NIC can calculate and obey aggregated rate limits, too. In the future, it can be changed, so that multiple SQs will back a single leaf class. ndo_select_queue is responsible for selecting the right queue that serves the traffic class of each packet. The data path works as follows: a packet is classified by clsact, the driver selects a hardware queue according to its class, and the packet is enqueued into this queue's qdisc. This solution addresses two main problems of scaling HTB: 1. Contention by flow classification. Currently the filters are attached to the HTB instance as follows: # tc filter add dev eth0 parent 1:0 protocol ip flower dst_port 80 classid 1:10 It's possible to move classification to clsact egress hook, which is thread-safe and lock-free: # tc filter add dev eth0 egress protocol ip flower dst_port 80 action skbedit priority 1:10 This way classification still happens in software, but the lock contention is eliminated, and it happens before selecting the TX queue, allowing the driver to translate the class to the corresponding hardware queue in ndo_select_queue. Note that this is already compatible with non-offloaded HTB and doesn't require changes to the kernel nor iproute2. 2. Contention by handling packets. HTB is not multi-queue, it attaches to a whole net device, and handling of all packets takes the same lock. When HTB is offloaded, it registers itself as a multi-queue qdisc, similarly to mq: HTB is attached to the netdev, and each queue has its own qdisc. Some features of HTB may be not supported by some particular hardware, for example, the maximum number of classes may be limited, the granularity of rate and ceil parameters may be different, etc. - so, the offload is not enabled by default, a new parameter is used to enable it: # tc qdisc replace dev eth0 root handle 1: htb offload Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 1 + include/net/pkt_cls.h | 36 +++ include/uapi/linux/pkt_sched.h | 1 + net/sched/sch_htb.c | 501 +++++++++++++++++++++++++++++++++-- tools/include/uapi/linux/pkt_sched.h | 1 + 5 files changed, 512 insertions(+), 28 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index ef517254367d..9e8572533d8e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -858,6 +858,7 @@ enum tc_setup_type { TC_SETUP_QDISC_ETS, TC_SETUP_QDISC_TBF, TC_SETUP_QDISC_FIFO, + TC_SETUP_QDISC_HTB, }; /* These structures hold the attributes of bpf state that are being passed diff --git a/include/net/pkt_cls.h b/include/net/pkt_cls.h index 0f2a9c44171c..255e4f4b521f 100644 --- a/include/net/pkt_cls.h +++ b/include/net/pkt_cls.h @@ -783,6 +783,42 @@ struct tc_mq_qopt_offload { }; }; +enum tc_htb_command { + /* Root */ + TC_HTB_CREATE, /* Initialize HTB offload. */ + TC_HTB_DESTROY, /* Destroy HTB offload. */ + + /* Classes */ + /* Allocate qid and create leaf. */ + TC_HTB_LEAF_ALLOC_QUEUE, + /* Convert leaf to inner, preserve and return qid, create new leaf. */ + TC_HTB_LEAF_TO_INNER, + /* Delete leaf, while siblings remain. */ + TC_HTB_LEAF_DEL, + /* Delete leaf, convert parent to leaf, preserving qid. */ + TC_HTB_LEAF_DEL_LAST, + /* TC_HTB_LEAF_DEL_LAST, but delete driver data on hardware errors. */ + TC_HTB_LEAF_DEL_LAST_FORCE, + /* Modify parameters of a node. */ + TC_HTB_NODE_MODIFY, + + /* Class qdisc */ + TC_HTB_LEAF_QUERY_QUEUE, /* Query qid by classid. */ +}; + +struct tc_htb_qopt_offload { + struct netlink_ext_ack *extack; + enum tc_htb_command command; + u16 classid; + u32 parent_classid; + u16 qid; + u16 moved_qid; + u64 rate; + u64 ceil; +}; + +#define TC_HTB_CLASSID_ROOT U32_MAX + enum tc_red_command { TC_RED_REPLACE, TC_RED_DESTROY, diff --git a/include/uapi/linux/pkt_sched.h b/include/uapi/linux/pkt_sched.h index 9e7c2c607845..79a699f106b1 100644 --- a/include/uapi/linux/pkt_sched.h +++ b/include/uapi/linux/pkt_sched.h @@ -434,6 +434,7 @@ enum { TCA_HTB_RATE64, TCA_HTB_CEIL64, TCA_HTB_PAD, + TCA_HTB_OFFLOAD, __TCA_HTB_MAX, }; diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index a8fc97b05bd8..d1b60fe3d311 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -174,6 +174,11 @@ struct htb_sched { int row_mask[TC_HTB_MAXDEPTH]; struct htb_level hlevel[TC_HTB_MAXDEPTH]; + + struct Qdisc **direct_qdiscs; + unsigned int num_direct_qdiscs; + + bool offload; }; /* find class in global hash table using given handle */ @@ -957,7 +962,7 @@ static void htb_reset(struct Qdisc *sch) if (cl->level) memset(&cl->inner, 0, sizeof(cl->inner)); else { - if (cl->leaf.q) + if (cl->leaf.q && !q->offload) qdisc_reset(cl->leaf.q); } cl->prio_activity = 0; @@ -980,6 +985,7 @@ static const struct nla_policy htb_policy[TCA_HTB_MAX + 1] = { [TCA_HTB_DIRECT_QLEN] = { .type = NLA_U32 }, [TCA_HTB_RATE64] = { .type = NLA_U64 }, [TCA_HTB_CEIL64] = { .type = NLA_U64 }, + [TCA_HTB_OFFLOAD] = { .type = NLA_FLAG }, }; static void htb_work_func(struct work_struct *work) @@ -992,12 +998,27 @@ static void htb_work_func(struct work_struct *work) rcu_read_unlock(); } +static void htb_set_lockdep_class_child(struct Qdisc *q) +{ + static struct lock_class_key child_key; + + lockdep_set_class(qdisc_lock(q), &child_key); +} + +static int htb_offload(struct net_device *dev, struct tc_htb_qopt_offload *opt) +{ + return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_HTB, opt); +} + static int htb_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { + struct net_device *dev = qdisc_dev(sch); + struct tc_htb_qopt_offload offload_opt; struct htb_sched *q = qdisc_priv(sch); struct nlattr *tb[TCA_HTB_MAX + 1]; struct tc_htb_glob *gopt; + unsigned int ntx; int err; qdisc_watchdog_init(&q->watchdog, sch); @@ -1022,9 +1043,26 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt, if (gopt->version != HTB_VER >> 16) return -EINVAL; + q->offload = nla_get_flag(tb[TCA_HTB_OFFLOAD]); + + if (q->offload) { + if (sch->parent != TC_H_ROOT) + return -EOPNOTSUPP; + + if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) + return -EOPNOTSUPP; + + q->num_direct_qdiscs = dev->real_num_tx_queues; + q->direct_qdiscs = kcalloc(q->num_direct_qdiscs, + sizeof(*q->direct_qdiscs), + GFP_KERNEL); + if (!q->direct_qdiscs) + return -ENOMEM; + } + err = qdisc_class_hash_init(&q->clhash); if (err < 0) - return err; + goto err_free_direct_qdiscs; qdisc_skb_head_init(&q->direct_queue); @@ -1037,7 +1075,107 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt, q->rate2quantum = 1; q->defcls = gopt->defcls; + if (!q->offload) + return 0; + + for (ntx = 0; ntx < q->num_direct_qdiscs; ntx++) { + struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, ntx); + struct Qdisc *qdisc; + + qdisc = qdisc_create_dflt(dev_queue, &pfifo_qdisc_ops, + TC_H_MAKE(sch->handle, 0), extack); + if (!qdisc) { + err = -ENOMEM; + goto err_free_qdiscs; + } + + htb_set_lockdep_class_child(qdisc); + q->direct_qdiscs[ntx] = qdisc; + qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT; + } + + sch->flags |= TCQ_F_MQROOT; + + offload_opt = (struct tc_htb_qopt_offload) { + .command = TC_HTB_CREATE, + .parent_classid = TC_H_MAJ(sch->handle) >> 16, + .classid = TC_H_MIN(q->defcls), + .extack = extack, + }; + err = htb_offload(dev, &offload_opt); + if (err) + goto err_free_qdiscs; + return 0; + +err_free_qdiscs: + /* TC_HTB_CREATE call failed, avoid any further calls to the driver. */ + q->offload = false; + + for (ntx = 0; ntx < q->num_direct_qdiscs && q->direct_qdiscs[ntx]; + ntx++) + qdisc_put(q->direct_qdiscs[ntx]); + + qdisc_class_hash_destroy(&q->clhash); + /* Prevent use-after-free and double-free when htb_destroy gets called. + */ + q->clhash.hash = NULL; + q->clhash.hashsize = 0; + +err_free_direct_qdiscs: + kfree(q->direct_qdiscs); + q->direct_qdiscs = NULL; + return err; +} + +static void htb_attach_offload(struct Qdisc *sch) +{ + struct net_device *dev = qdisc_dev(sch); + struct htb_sched *q = qdisc_priv(sch); + unsigned int ntx; + + for (ntx = 0; ntx < q->num_direct_qdiscs; ntx++) { + struct Qdisc *old, *qdisc = q->direct_qdiscs[ntx]; + + old = dev_graft_qdisc(qdisc->dev_queue, qdisc); + qdisc_put(old); + qdisc_hash_add(qdisc, false); + } + for (ntx = q->num_direct_qdiscs; ntx < dev->num_tx_queues; ntx++) { + struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, ntx); + struct Qdisc *old = dev_graft_qdisc(dev_queue, NULL); + + qdisc_put(old); + } + + kfree(q->direct_qdiscs); + q->direct_qdiscs = NULL; +} + +static void htb_attach_software(struct Qdisc *sch) +{ + struct net_device *dev = qdisc_dev(sch); + unsigned int ntx; + + /* Resemble qdisc_graft behavior. */ + for (ntx = 0; ntx < dev->num_tx_queues; ntx++) { + struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, ntx); + struct Qdisc *old = dev_graft_qdisc(dev_queue, sch); + + qdisc_refcount_inc(sch); + + qdisc_put(old); + } +} + +static void htb_attach(struct Qdisc *sch) +{ + struct htb_sched *q = qdisc_priv(sch); + + if (q->offload) + htb_attach_offload(sch); + else + htb_attach_software(sch); } static int htb_dump(struct Qdisc *sch, struct sk_buff *skb) @@ -1046,6 +1184,11 @@ static int htb_dump(struct Qdisc *sch, struct sk_buff *skb) struct nlattr *nest; struct tc_htb_glob gopt; + if (q->offload) + sch->flags |= TCQ_F_OFFLOADED; + else + sch->flags &= ~TCQ_F_OFFLOADED; + sch->qstats.overlimits = q->overlimits; /* Its safe to not acquire qdisc lock. As we hold RTNL, * no change can happen on the qdisc parameters. @@ -1063,6 +1206,8 @@ static int htb_dump(struct Qdisc *sch, struct sk_buff *skb) if (nla_put(skb, TCA_HTB_INIT, sizeof(gopt), &gopt) || nla_put_u32(skb, TCA_HTB_DIRECT_QLEN, q->direct_qlen)) goto nla_put_failure; + if (q->offload && nla_put_flag(skb, TCA_HTB_OFFLOAD)) + goto nla_put_failure; return nla_nest_end(skb, nest); @@ -1144,19 +1289,97 @@ htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d) return gnet_stats_copy_app(d, &cl->xstats, sizeof(cl->xstats)); } +static struct netdev_queue * +htb_select_queue(struct Qdisc *sch, struct tcmsg *tcm) +{ + struct net_device *dev = qdisc_dev(sch); + struct tc_htb_qopt_offload offload_opt; + int err; + + offload_opt = (struct tc_htb_qopt_offload) { + .command = TC_HTB_LEAF_QUERY_QUEUE, + .classid = TC_H_MIN(tcm->tcm_parent), + }; + err = htb_offload(dev, &offload_opt); + if (err || offload_opt.qid >= dev->num_tx_queues) + return NULL; + return netdev_get_tx_queue(dev, offload_opt.qid); +} + +static struct Qdisc * +htb_graft_helper(struct netdev_queue *dev_queue, struct Qdisc *new_q) +{ + struct net_device *dev = dev_queue->dev; + struct Qdisc *old_q; + + if (dev->flags & IFF_UP) + dev_deactivate(dev); + old_q = dev_graft_qdisc(dev_queue, new_q); + if (new_q) + new_q->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT; + if (dev->flags & IFF_UP) + dev_activate(dev); + + return old_q; +} + +static void htb_offload_move_qdisc(struct Qdisc *sch, u16 qid_old, u16 qid_new) +{ + struct netdev_queue *queue_old, *queue_new; + struct net_device *dev = qdisc_dev(sch); + struct Qdisc *qdisc; + + queue_old = netdev_get_tx_queue(dev, qid_old); + queue_new = netdev_get_tx_queue(dev, qid_new); + + if (dev->flags & IFF_UP) + dev_deactivate(dev); + qdisc = dev_graft_qdisc(queue_old, NULL); + qdisc->dev_queue = queue_new; + qdisc = dev_graft_qdisc(queue_new, qdisc); + if (dev->flags & IFF_UP) + dev_activate(dev); + + WARN_ON(!(qdisc->flags & TCQ_F_BUILTIN)); +} + static int htb_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new, struct Qdisc **old, struct netlink_ext_ack *extack) { + struct netdev_queue *dev_queue = sch->dev_queue; struct htb_class *cl = (struct htb_class *)arg; + struct htb_sched *q = qdisc_priv(sch); + struct Qdisc *old_q; if (cl->level) return -EINVAL; - if (new == NULL && - (new = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, - cl->common.classid, extack)) == NULL) - return -ENOBUFS; + + if (q->offload) { + dev_queue = new->dev_queue; + WARN_ON(dev_queue != cl->leaf.q->dev_queue); + } + + if (!new) { + new = qdisc_create_dflt(dev_queue, &pfifo_qdisc_ops, + cl->common.classid, extack); + if (!new) + return -ENOBUFS; + } + + if (q->offload) { + htb_set_lockdep_class_child(new); + /* One ref for cl->leaf.q, the other for dev_queue->qdisc. */ + qdisc_refcount_inc(new); + old_q = htb_graft_helper(dev_queue, new); + } *old = qdisc_replace(sch, new, &cl->leaf.q); + + if (q->offload) { + WARN_ON(old_q != *old); + qdisc_put(old_q); + } + return 0; } @@ -1184,9 +1407,10 @@ static inline int htb_parent_last_child(struct htb_class *cl) return 1; } -static void htb_parent_to_leaf(struct htb_sched *q, struct htb_class *cl, +static void htb_parent_to_leaf(struct Qdisc *sch, struct htb_class *cl, struct Qdisc *new_q) { + struct htb_sched *q = qdisc_priv(sch); struct htb_class *parent = cl->parent; WARN_ON(cl->level || !cl->leaf.q || cl->prio_activity); @@ -1204,6 +1428,71 @@ static void htb_parent_to_leaf(struct htb_sched *q, struct htb_class *cl, parent->cmode = HTB_CAN_SEND; } +static void htb_parent_to_leaf_offload(struct Qdisc *sch, + struct netdev_queue *dev_queue, + struct Qdisc *new_q) +{ + struct Qdisc *old_q; + + /* One ref for cl->leaf.q, the other for dev_queue->qdisc. */ + qdisc_refcount_inc(new_q); + old_q = htb_graft_helper(dev_queue, new_q); + WARN_ON(!(old_q->flags & TCQ_F_BUILTIN)); +} + +static int htb_destroy_class_offload(struct Qdisc *sch, struct htb_class *cl, + bool last_child, bool destroying, + struct netlink_ext_ack *extack) +{ + struct tc_htb_qopt_offload offload_opt; + struct Qdisc *q = cl->leaf.q; + struct Qdisc *old = NULL; + int err; + + if (cl->level) + return -EINVAL; + + WARN_ON(!q); + if (!destroying) { + /* On destroy of HTB, two cases are possible: + * 1. q is a normal qdisc, but q->dev_queue has noop qdisc. + * 2. q is a noop qdisc (for nodes that were inner), + * q->dev_queue is noop_netdev_queue. + */ + old = htb_graft_helper(q->dev_queue, NULL); + WARN_ON(!old); + WARN_ON(old != q); + } + + offload_opt = (struct tc_htb_qopt_offload) { + .command = !last_child ? TC_HTB_LEAF_DEL : + destroying ? TC_HTB_LEAF_DEL_LAST_FORCE : + TC_HTB_LEAF_DEL_LAST, + .classid = cl->common.classid, + .extack = extack, + }; + err = htb_offload(qdisc_dev(sch), &offload_opt); + + if (!err || destroying) + qdisc_put(old); + else + htb_graft_helper(q->dev_queue, old); + + if (last_child) + return err; + + if (!err && offload_opt.moved_qid != 0) { + if (destroying) + q->dev_queue = netdev_get_tx_queue(qdisc_dev(sch), + offload_opt.qid); + else + htb_offload_move_qdisc(sch, offload_opt.moved_qid, + offload_opt.qid); + } + + return err; +} + static void htb_destroy_class(struct Qdisc *sch, struct htb_class *cl) { if (!cl->level) { @@ -1217,8 +1506,11 @@ static void htb_destroy_class(struct Qdisc *sch, struct htb_class *cl) static void htb_destroy(struct Qdisc *sch) { + struct net_device *dev = qdisc_dev(sch); + struct tc_htb_qopt_offload offload_opt; struct htb_sched *q = qdisc_priv(sch); struct hlist_node *next; + bool nonempty, changed; struct htb_class *cl; unsigned int i; @@ -1237,13 +1529,58 @@ static void htb_destroy(struct Qdisc *sch) cl->block = NULL; } } - for (i = 0; i < q->clhash.hashsize; i++) { - hlist_for_each_entry_safe(cl, next, &q->clhash.hash[i], - common.hnode) - htb_destroy_class(sch, cl); - } + + do { + nonempty = false; + changed = false; + for (i = 0; i < q->clhash.hashsize; i++) { + hlist_for_each_entry_safe(cl, next, &q->clhash.hash[i], + common.hnode) { + bool last_child; + + if (!q->offload) { + htb_destroy_class(sch, cl); + continue; + } + + nonempty = true; + + if (cl->level) + continue; + + changed = true; + + last_child = htb_parent_last_child(cl); + htb_destroy_class_offload(sch, cl, last_child, + true, NULL); + qdisc_class_hash_remove(&q->clhash, + &cl->common); + if (cl->parent) + cl->parent->children--; + if (last_child) + htb_parent_to_leaf(sch, cl, NULL); + htb_destroy_class(sch, cl); + } + } + } while (changed); + WARN_ON(nonempty); + qdisc_class_hash_destroy(&q->clhash); __qdisc_reset_queue(&q->direct_queue); + + if (!q->offload) + return; + + offload_opt = (struct tc_htb_qopt_offload) { + .command = TC_HTB_DESTROY, + }; + htb_offload(dev, &offload_opt); + + if (!q->direct_qdiscs) + return; + for (i = 0; i < q->num_direct_qdiscs && q->direct_qdiscs[i]; i++) + qdisc_put(q->direct_qdiscs[i]); + kfree(q->direct_qdiscs); } static int htb_delete(struct Qdisc *sch, unsigned long arg, @@ -1253,6 +1590,7 @@ static int htb_delete(struct Qdisc *sch, unsigned long arg, struct htb_class *cl = (struct htb_class *)arg; struct Qdisc *new_q = NULL; int last_child = 0; + int err; /* TODO: why don't allow to delete subtree ? references ? does * tc subsys guarantee us that in htb_destroy it holds no class @@ -1261,11 +1599,28 @@ static int htb_delete(struct Qdisc *sch, unsigned long arg, if (cl->children || cl->filter_cnt) return -EBUSY; - if (!cl->level && htb_parent_last_child(cl)) { - new_q = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, + if (!cl->level && htb_parent_last_child(cl)) + last_child = 1; + + if (q->offload) { + err = htb_destroy_class_offload(sch, cl, last_child, false, + extack); + if (err) + return err; + } + + if (last_child) { + struct netdev_queue *dev_queue; + + dev_queue = q->offload ? cl->leaf.q->dev_queue : sch->dev_queue; + new_q = qdisc_create_dflt(dev_queue, &pfifo_qdisc_ops, cl->parent->common.classid, NULL); - last_child = 1; + if (q->offload) { + if (new_q) + htb_set_lockdep_class_child(new_q); + htb_parent_to_leaf_offload(sch, dev_queue, new_q); + } } sch_tree_lock(sch); @@ -1286,7 +1641,7 @@ static int htb_delete(struct Qdisc *sch, unsigned long arg, &q->hlevel[cl->level].wait_pq); if (last_child) - htb_parent_to_leaf(q, cl, new_q); + htb_parent_to_leaf(sch, cl, new_q); sch_tree_unlock(sch); @@ -1301,9 +1656,11 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, int err = -EINVAL; struct htb_sched *q = qdisc_priv(sch); struct htb_class *cl = (struct htb_class *)*arg, *parent; + struct tc_htb_qopt_offload offload_opt; struct nlattr *opt = tca[TCA_OPTIONS]; struct nlattr *tb[TCA_HTB_MAX + 1]; struct Qdisc *parent_qdisc = NULL; + struct netdev_queue *dev_queue; struct tc_htb_opt *hopt; u64 rate64, ceil64; int warn = 0; @@ -1336,8 +1693,12 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, qdisc_put_rtab(qdisc_get_rtab(&hopt->ceil, tb[TCA_HTB_CTAB], NULL)); + rate64 = tb[TCA_HTB_RATE64] ? nla_get_u64(tb[TCA_HTB_RATE64]) : 0; + ceil64 = tb[TCA_HTB_CEIL64] ? nla_get_u64(tb[TCA_HTB_CEIL64]) : 0; + if (!cl) { /* new class */ - struct Qdisc *new_q; + struct net_device *dev = qdisc_dev(sch); + struct Qdisc *new_q, *old_q; int prio; struct { struct nlattr nla; @@ -1380,11 +1741,8 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, NULL, qdisc_root_sleeping_running(sch), tca[TCA_RATE] ? : &est.nla); - if (err) { - tcf_block_put(cl->block); - kfree(cl); - goto failure; - } + if (err) + goto err_block_put; } cl->children = 0; @@ -1393,12 +1751,74 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, for (prio = 0; prio < TC_HTB_NUMPRIO; prio++) RB_CLEAR_NODE(&cl->node[prio]); + cl->common.classid = classid; + + /* Make sure nothing interrupts us in between of two + * ndo_setup_tc calls. + */ + ASSERT_RTNL(); + /* create leaf qdisc early because it uses kmalloc(GFP_KERNEL) * so that can't be used inside of sch_tree_lock * -- thanks to Karlis Peisenieks */ - new_q = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, + if (!q->offload) { + dev_queue = sch->dev_queue; + } else if (!(parent && !parent->level)) { + /* Assign a dev_queue to this classid. */ + offload_opt = (struct tc_htb_qopt_offload) { + .command = TC_HTB_LEAF_ALLOC_QUEUE, + .classid = cl->common.classid, + .parent_classid = parent ? + TC_H_MIN(parent->common.classid) : + TC_HTB_CLASSID_ROOT, + .rate = max_t(u64, hopt->rate.rate, rate64), + .ceil = max_t(u64, hopt->ceil.rate, ceil64), + .extack = extack, + }; + err = htb_offload(dev, &offload_opt); + if (err) { + pr_err("htb: TC_HTB_LEAF_ALLOC_QUEUE failed with err = %d\n", + err); + goto err_kill_estimator; + } + dev_queue = netdev_get_tx_queue(dev, offload_opt.qid); + } else { /* First child. */ + dev_queue = parent->leaf.q->dev_queue; + old_q = htb_graft_helper(dev_queue, NULL); + WARN_ON(old_q != parent->leaf.q); + offload_opt = (struct tc_htb_qopt_offload) { + .command = TC_HTB_LEAF_TO_INNER, + .classid = cl->common.classid, + .parent_classid = + TC_H_MIN(parent->common.classid), + .rate = max_t(u64, hopt->rate.rate, rate64), + .ceil = max_t(u64, hopt->ceil.rate, ceil64), + .extack = extack, + }; + err = htb_offload(dev, &offload_opt); + if (err) { + pr_err("htb: TC_HTB_LEAF_TO_INNER failed with err = %d\n", + err); + htb_graft_helper(dev_queue, old_q); + goto err_kill_estimator; + } + qdisc_put(old_q); + } + new_q = qdisc_create_dflt(dev_queue, &pfifo_qdisc_ops, classid, NULL); + if (q->offload) { + if (new_q) { + htb_set_lockdep_class_child(new_q); + /* One ref for cl->leaf.q, the other for + * dev_queue->qdisc. + */ + qdisc_refcount_inc(new_q); + } + old_q = htb_graft_helper(dev_queue, new_q); + /* No qdisc_put needed. */ + WARN_ON(!(old_q->flags & TCQ_F_BUILTIN)); + } sch_tree_lock(sch); if (parent && !parent->level) { /* turn parent into inner node */ @@ -1416,10 +1836,10 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, : TC_HTB_MAXDEPTH) - 1; memset(&parent->inner, 0, sizeof(parent->inner)); } + /* leaf (we) needs elementary qdisc */ cl->leaf.q = new_q ? new_q : &noop_qdisc; - cl->common.classid = classid; cl->parent = parent; /* set class to be in HTB_CAN_SEND state */ @@ -1445,12 +1865,30 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, if (err) return err; } - sch_tree_lock(sch); - } - rate64 = tb[TCA_HTB_RATE64] ? nla_get_u64(tb[TCA_HTB_RATE64]) : 0; + if (q->offload) { + struct net_device *dev = qdisc_dev(sch); + + offload_opt = (struct tc_htb_qopt_offload) { + .command = TC_HTB_NODE_MODIFY, + .classid = cl->common.classid, + .rate = max_t(u64, hopt->rate.rate, rate64), + .ceil = max_t(u64, hopt->ceil.rate, ceil64), + .extack = extack, + }; + err = htb_offload(dev, &offload_opt); + if (err) + /* Estimator was replaced, and rollback may fail + * as well, so we don't try to recover it, and + * the estimator won't work property with the + * offload anyway, because bstats are updated + * only when the stats are queried. + */ + return err; + } - ceil64 = tb[TCA_HTB_CEIL64] ? nla_get_u64(tb[TCA_HTB_CEIL64]) : 0; + sch_tree_lock(sch); + } psched_ratecfg_precompute(&cl->rate, &hopt->rate, rate64); psched_ratecfg_precompute(&cl->ceil, &hopt->ceil, ceil64); @@ -1493,6 +1931,11 @@ static int htb_change_class(struct Qdisc *sch, u32 classid, *arg = (unsigned long)cl; return 0; +err_kill_estimator: + gen_kill_estimator(&cl->rate_est); +err_block_put: + tcf_block_put(cl->block); + kfree(cl); failure: return err; } @@ -1558,6 +2001,7 @@ static void htb_walk(struct Qdisc *sch, struct qdisc_walker *arg) } static const struct Qdisc_class_ops htb_class_ops = { + .select_queue = htb_select_queue, .graft = htb_graft, .leaf = htb_leaf, .qlen_notify = htb_qlen_notify, @@ -1580,6 +2024,7 @@ static struct Qdisc_ops htb_qdisc_ops __read_mostly = { .dequeue = htb_dequeue, .peek = qdisc_peek_dequeued, .init = htb_init, + .attach = htb_attach, .reset = htb_reset, .destroy = htb_destroy, .dump = htb_dump, diff --git a/tools/include/uapi/linux/pkt_sched.h b/tools/include/uapi/linux/pkt_sched.h index 0d18b1d1fbbc..5c903abc9fa5 100644 --- a/tools/include/uapi/linux/pkt_sched.h +++ b/tools/include/uapi/linux/pkt_sched.h @@ -414,6 +414,7 @@ enum { TCA_HTB_RATE64, TCA_HTB_CEIL64, TCA_HTB_PAD, + TCA_HTB_OFFLOAD, __TCA_HTB_MAX, }; -- cgit v1.2.3 From 214baf22870cfa437522f3bd4fbae56338674b04 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Tue, 19 Jan 2021 14:08:15 +0200 Subject: net/mlx5e: Support HTB offload This commit adds support for HTB offload in the mlx5e driver. Performance: NIC: Mellanox ConnectX-6 Dx CPU: Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz (24 cores with HT) 100 Gbit/s line rate, 500 UDP streams @ ~200 Mbit/s each 48 traffic classes, flower used for steering No shaping (rate limits set to 4 Gbit/s per TC) - checking for max throughput. Baseline: 98.7 Gbps, 8.25 Mpps HTB: 6.7 Gbps, 0.56 Mpps HTB offload: 95.6 Gbps, 8.00 Mpps Limitations: 1. 256 leaf nodes, 3 levels of depth. 2. Granularity for ceil is 1 Mbit/s. Rates are converted to weights, and the bandwidth is split among the siblings according to these weights. Other parameters for classes are not supported. Ethtool statistics support for QoS SQs are also added. The counters are called qos_txN_*, where N is the QoS queue number (starting from 0, the numeration is separate from the normal SQs), and * is the counter name (the counters are the same as for the normal SQs). Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/Makefile | 6 +- drivers/net/ethernet/mellanox/mlx5/core/en.h | 27 +- .../net/ethernet/mellanox/mlx5/core/en/params.h | 2 + drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/en/qos.c | 984 +++++++++++++++++++++ drivers/net/ethernet/mellanox/mlx5/core/en/qos.h | 44 + .../net/ethernet/mellanox/mlx5/core/en_ethtool.c | 21 + drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 176 +++- drivers/net/ethernet/mellanox/mlx5/core/en_stats.c | 100 +++ drivers/net/ethernet/mellanox/mlx5/core/en_stats.h | 2 + drivers/net/ethernet/mellanox/mlx5/core/en_tx.c | 47 +- drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c | 26 + drivers/net/ethernet/mellanox/mlx5/core/qos.c | 85 ++ drivers/net/ethernet/mellanox/mlx5/core/qos.h | 30 + include/linux/mlx5/mlx5_ifc.h | 13 +- 15 files changed, 1516 insertions(+), 49 deletions(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/qos.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/qos.h create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/qos.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/qos.h (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 134bd038ae8a..fcfc0b114985 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -16,7 +16,8 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ transobj.o vport.o sriov.o fs_cmd.o fs_core.o pci_irq.o \ fs_counters.o rl.o lag.o dev.o events.o wq.o lib/gid.o \ lib/devcom.o lib/pci_vsc.o lib/dm.o diag/fs_tracepoint.o \ - diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o fw_reset.o + diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o \ + fw_reset.o qos.o # # Netdev basic @@ -25,7 +26,8 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \ en_tx.o en_rx.o en_dim.o en_txrx.o en/xdp.o en_stats.o \ en_selftest.o en/port.o en/monitor_stats.o en/health.o \ en/reporter_tx.o en/reporter_rx.o en/params.o en/xsk/pool.o \ - en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o en/ptp.o + en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o en/ptp.o \ + en/qos.o # # Netdev extra diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 055baf3b6cb1..26e578a973e5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -55,6 +55,7 @@ #include "en_stats.h" #include "en/dcbnl.h" #include "en/fs.h" +#include "en/qos.h" #include "lib/hv_vhca.h" extern const struct net_device_ops mlx5e_netdev_ops; @@ -161,6 +162,9 @@ do { \ ##__VA_ARGS__); \ } while (0) +#define mlx5e_state_dereference(priv, p) \ + rcu_dereference_protected((p), lockdep_is_held(&(priv)->state_lock)) + enum mlx5e_rq_group { MLX5E_RQ_GROUP_REGULAR, MLX5E_RQ_GROUP_XSK, @@ -663,11 +667,13 @@ struct mlx5e_channel { struct mlx5e_xdpsq rq_xdpsq; struct mlx5e_txqsq sq[MLX5E_MAX_NUM_TC]; struct mlx5e_icosq icosq; /* internal control operations */ + struct mlx5e_txqsq __rcu * __rcu *qos_sqs; bool xdp; struct napi_struct napi; struct device *pdev; struct net_device *netdev; __be32 mkey_be; + u16 qos_sqs_size; u8 num_tc; u8 lag_port; @@ -756,6 +762,8 @@ struct mlx5e_modify_sq_param { int next_state; int rl_update; int rl_index; + bool qos_update; + u16 qos_queue_group_id; }; #if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE) @@ -788,10 +796,20 @@ struct mlx5e_scratchpad { cpumask_var_t cpumask; }; +struct mlx5e_htb { + DECLARE_HASHTABLE(qos_tc2node, order_base_2(MLX5E_QOS_MAX_LEAF_NODES)); + DECLARE_BITMAP(qos_used_qids, MLX5E_QOS_MAX_LEAF_NODES); + struct mlx5e_sq_stats **qos_sq_stats; + u16 max_qos_sqs; + u16 maj_id; + u16 defcls; +}; + struct mlx5e_priv { /* priv data path fields - start */ /* +1 for port ptp ts */ - struct mlx5e_txqsq *txq2sq[(MLX5E_MAX_NUM_CHANNELS + 1) * MLX5E_MAX_NUM_TC]; + struct mlx5e_txqsq *txq2sq[(MLX5E_MAX_NUM_CHANNELS + 1) * MLX5E_MAX_NUM_TC + + MLX5E_QOS_MAX_LEAF_NODES]; int channel_tc2realtxq[MLX5E_MAX_NUM_CHANNELS][MLX5E_MAX_NUM_TC]; int port_ptp_tc2realtxq[MLX5E_MAX_NUM_TC]; #ifdef CONFIG_MLX5_CORE_EN_DCB @@ -859,6 +877,7 @@ struct mlx5e_priv { struct mlx5e_hv_vhca_stats_agent stats_agent; #endif struct mlx5e_scratchpad scratchpad; + struct mlx5e_htb htb; }; struct mlx5e_rx_handlers { @@ -986,6 +1005,7 @@ int mlx5e_safe_switch_channels(struct mlx5e_priv *priv, struct mlx5e_channels *new_chs, mlx5e_fp_preactivate preactivate, void *context); +int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv); int mlx5e_num_channels_changed(struct mlx5e_priv *priv); int mlx5e_num_channels_changed_ctx(struct mlx5e_priv *priv, void *context); void mlx5e_activate_priv_channels(struct mlx5e_priv *priv); @@ -1010,6 +1030,9 @@ void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq); int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn, struct mlx5e_modify_sq_param *p); +int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix, + struct mlx5e_params *params, struct mlx5e_sq_param *param, + struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id, u16 qos_qid); void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq); void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq); void mlx5e_free_txqsq(struct mlx5e_txqsq *sq); @@ -1020,8 +1043,10 @@ struct mlx5e_create_sq_param; int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, struct mlx5e_sq_param *param, struct mlx5e_create_sq_param *csp, + u16 qos_queue_group_id, u32 *sqn); void mlx5e_tx_err_cqe_work(struct work_struct *recover_work); +void mlx5e_close_txqsq(struct mlx5e_txqsq *sq); static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h index 807147d97a0f..ea2cfb04b31a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -118,6 +118,8 @@ void mlx5e_build_rq_param(struct mlx5e_priv *priv, struct mlx5e_rq_param *param); void mlx5e_build_sq_param_common(struct mlx5e_priv *priv, struct mlx5e_sq_param *param); +void mlx5e_build_sq_param(struct mlx5e_priv *priv, struct mlx5e_params *params, + struct mlx5e_sq_param *param); void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index 2a2bac30daaa..eeddd1137dda 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -261,7 +261,7 @@ static int mlx5e_ptp_open_txqsq(struct mlx5e_port_ptp *c, u32 tisn, csp.min_inline_mode = txqsq->min_inline_mode; csp.ts_cqe_to_dest_cqn = ptpsq->ts_cq.mcq.cqn; - err = mlx5e_create_sq_rdy(c->mdev, sqp, &csp, &txqsq->sqn); + err = mlx5e_create_sq_rdy(c->mdev, sqp, &csp, 0, &txqsq->sqn); if (err) goto err_free_txqsq; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c new file mode 100644 index 000000000000..12d7ad061237 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c @@ -0,0 +1,984 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#include "en.h" +#include "params.h" +#include "../qos.h" + +#define BYTES_IN_MBIT 125000 + +int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev) +{ + return min(MLX5E_QOS_MAX_LEAF_NODES, mlx5_qos_max_leaf_nodes(mdev)); +} + +int mlx5e_qos_cur_leaf_nodes(struct mlx5e_priv *priv) +{ + int last = find_last_bit(priv->htb.qos_used_qids, mlx5e_qos_max_leaf_nodes(priv->mdev)); + + return last == mlx5e_qos_max_leaf_nodes(priv->mdev) ? 0 : last + 1; +} + +/* Software representation of the QoS tree (internal to this file) */ + +static int mlx5e_find_unused_qos_qid(struct mlx5e_priv *priv) +{ + int size = mlx5e_qos_max_leaf_nodes(priv->mdev); + int res; + + WARN_ONCE(!mutex_is_locked(&priv->state_lock), "%s: state_lock is not held\n", __func__); + res = find_first_zero_bit(priv->htb.qos_used_qids, size); + + return res == size ? -ENOSPC : res; +} + +struct mlx5e_qos_node { + struct hlist_node hnode; + struct rcu_head rcu; + struct mlx5e_qos_node *parent; + u64 rate; + u32 bw_share; + u32 max_average_bw; + u32 hw_id; + u32 classid; /* 16-bit, except root. */ + u16 qid; +}; + +#define MLX5E_QOS_QID_INNER 0xffff +#define MLX5E_HTB_CLASSID_ROOT 0xffffffff + +static struct mlx5e_qos_node * +mlx5e_sw_node_create_leaf(struct mlx5e_priv *priv, u16 classid, u16 qid, + struct mlx5e_qos_node *parent) +{ + struct mlx5e_qos_node *node; + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return ERR_PTR(-ENOMEM); + + node->parent = parent; + + node->qid = qid; + __set_bit(qid, priv->htb.qos_used_qids); + + node->classid = classid; + hash_add_rcu(priv->htb.qos_tc2node, &node->hnode, classid); + + mlx5e_update_tx_netdev_queues(priv); + + return node; +} + +static struct mlx5e_qos_node *mlx5e_sw_node_create_root(struct mlx5e_priv *priv) +{ + struct mlx5e_qos_node *node; + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return ERR_PTR(-ENOMEM); + + node->qid = MLX5E_QOS_QID_INNER; + node->classid = MLX5E_HTB_CLASSID_ROOT; + hash_add_rcu(priv->htb.qos_tc2node, &node->hnode, node->classid); + + return node; +} + +static struct mlx5e_qos_node *mlx5e_sw_node_find(struct mlx5e_priv *priv, u32 classid) +{ + struct mlx5e_qos_node *node = NULL; + + hash_for_each_possible(priv->htb.qos_tc2node, node, hnode, classid) { + if (node->classid == classid) + break; + } + + return node; +} + +static struct mlx5e_qos_node *mlx5e_sw_node_find_rcu(struct mlx5e_priv *priv, u32 classid) +{ + struct mlx5e_qos_node *node = NULL; + + hash_for_each_possible_rcu(priv->htb.qos_tc2node, node, hnode, classid) { + if (node->classid == classid) + break; + } + + return node; +} + +static void mlx5e_sw_node_delete(struct mlx5e_priv *priv, struct mlx5e_qos_node *node) +{ + hash_del_rcu(&node->hnode); + if (node->qid != MLX5E_QOS_QID_INNER) { + __clear_bit(node->qid, priv->htb.qos_used_qids); + mlx5e_update_tx_netdev_queues(priv); + } + kfree_rcu(node, rcu); +} + +/* TX datapath API */ + +static u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) +{ + /* These channel params are safe to access from the datapath, because: + * 1. This function is called only after checking priv->htb.maj_id != 0, + * and the number of queues can't change while HTB offload is active. + * 2. When priv->htb.maj_id becomes 0, synchronize_rcu waits for + * mlx5e_select_queue to finish while holding priv->state_lock, + * preventing other code from changing the number of queues. + */ + bool is_ptp = MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS); + + return (chs->params.num_channels + is_ptp) * chs->params.num_tc + qid; +} + +int mlx5e_get_txq_by_classid(struct mlx5e_priv *priv, u16 classid) +{ + struct mlx5e_qos_node *node; + u16 qid; + int res; + + rcu_read_lock(); + + node = mlx5e_sw_node_find_rcu(priv, classid); + if (!node) { + res = -ENOENT; + goto out; + } + qid = READ_ONCE(node->qid); + if (qid == MLX5E_QOS_QID_INNER) { + res = -EINVAL; + goto out; + } + res = mlx5e_qid_from_qos(&priv->channels, qid); + +out: + rcu_read_unlock(); + return res; +} + +static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid) +{ + struct mlx5e_params *params = &priv->channels.params; + struct mlx5e_txqsq __rcu **qos_sqs; + struct mlx5e_channel *c; + int ix; + + ix = qid % params->num_channels; + qid /= params->num_channels; + c = priv->channels.c[ix]; + + qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); + return mlx5e_state_dereference(priv, qos_sqs[qid]); +} + +/* SQ lifecycle */ + +static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, + struct mlx5e_qos_node *node) +{ + struct mlx5e_create_cq_param ccp = {}; + struct mlx5e_txqsq __rcu **qos_sqs; + struct mlx5e_sq_param param_sq; + struct mlx5e_cq_param param_cq; + int txq_ix, ix, qid, err = 0; + struct mlx5e_params *params; + struct mlx5e_channel *c; + struct mlx5e_txqsq *sq; + + params = &chs->params; + + txq_ix = mlx5e_qid_from_qos(chs, node->qid); + + WARN_ON(node->qid > priv->htb.max_qos_sqs); + if (node->qid == priv->htb.max_qos_sqs) { + struct mlx5e_sq_stats *stats, **stats_list = NULL; + + if (priv->htb.max_qos_sqs == 0) { + stats_list = kvcalloc(mlx5e_qos_max_leaf_nodes(priv->mdev), + sizeof(*stats_list), + GFP_KERNEL); + if (!stats_list) + return -ENOMEM; + } + stats = kzalloc(sizeof(*stats), GFP_KERNEL); + if (!stats) { + kvfree(stats_list); + return -ENOMEM; + } + if (stats_list) + WRITE_ONCE(priv->htb.qos_sq_stats, stats_list); + WRITE_ONCE(priv->htb.qos_sq_stats[node->qid], stats); + /* Order max_qos_sqs increment after writing the array pointer. + * Pairs with smp_load_acquire in en_stats.c. + */ + smp_store_release(&priv->htb.max_qos_sqs, priv->htb.max_qos_sqs + 1); + } + + ix = node->qid % params->num_channels; + qid = node->qid / params->num_channels; + c = chs->c[ix]; + + qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); + sq = kzalloc(sizeof(*sq), GFP_KERNEL); + + if (!sq) + return -ENOMEM; + + mlx5e_build_create_cq_param(&ccp, c); + + memset(¶m_sq, 0, sizeof(param_sq)); + memset(¶m_cq, 0, sizeof(param_cq)); + mlx5e_build_sq_param(priv, params, ¶m_sq); + mlx5e_build_tx_cq_param(priv, params, ¶m_cq); + err = mlx5e_open_cq(priv, params->tx_cq_moderation, ¶m_cq, &ccp, &sq->cq); + if (err) + goto err_free_sq; + err = mlx5e_open_txqsq(c, priv->tisn[c->lag_port][0], txq_ix, params, + ¶m_sq, sq, 0, node->hw_id, node->qid); + if (err) + goto err_close_cq; + + rcu_assign_pointer(qos_sqs[qid], sq); + + return 0; + +err_close_cq: + mlx5e_close_cq(&sq->cq); +err_free_sq: + kfree(sq); + return err; +} + +static void mlx5e_activate_qos_sq(struct mlx5e_priv *priv, struct mlx5e_qos_node *node) +{ + struct mlx5e_txqsq *sq; + + sq = mlx5e_get_qos_sq(priv, node->qid); + + WRITE_ONCE(priv->txq2sq[mlx5e_qid_from_qos(&priv->channels, node->qid)], sq); + + /* Make the change to txq2sq visible before the queue is started. + * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE, + * which pairs with this barrier. + */ + smp_wmb(); + + qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node->qid); + mlx5e_activate_txqsq(sq); +} + +static void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) +{ + struct mlx5e_txqsq *sq; + + sq = mlx5e_get_qos_sq(priv, qid); + if (!sq) /* Handle the case when the SQ failed to open. */ + return; + + qos_dbg(priv->mdev, "Deactivate QoS SQ qid %u\n", qid); + mlx5e_deactivate_txqsq(sq); + + /* The queue is disabled, no synchronization with datapath is needed. */ + priv->txq2sq[mlx5e_qid_from_qos(&priv->channels, qid)] = NULL; +} + +static void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid) +{ + struct mlx5e_txqsq __rcu **qos_sqs; + struct mlx5e_params *params; + struct mlx5e_channel *c; + struct mlx5e_txqsq *sq; + int ix; + + params = &priv->channels.params; + + ix = qid % params->num_channels; + qid /= params->num_channels; + c = priv->channels.c[ix]; + qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); + sq = rcu_replace_pointer(qos_sqs[qid], NULL, lockdep_is_held(&priv->state_lock)); + if (!sq) /* Handle the case when the SQ failed to open. */ + return; + + synchronize_rcu(); /* Sync with NAPI. */ + + mlx5e_close_txqsq(sq); + mlx5e_close_cq(&sq->cq); + kfree(sq); +} + +void mlx5e_qos_close_queues(struct mlx5e_channel *c) +{ + struct mlx5e_txqsq __rcu **qos_sqs; + int i; + + qos_sqs = rcu_replace_pointer(c->qos_sqs, NULL, lockdep_is_held(&c->priv->state_lock)); + if (!qos_sqs) + return; + synchronize_rcu(); /* Sync with NAPI. */ + + for (i = 0; i < c->qos_sqs_size; i++) { + struct mlx5e_txqsq *sq; + + sq = mlx5e_state_dereference(c->priv, qos_sqs[i]); + if (!sq) /* Handle the case when the SQ failed to open. */ + continue; + + mlx5e_close_txqsq(sq); + mlx5e_close_cq(&sq->cq); + kfree(sq); + } + + kvfree(qos_sqs); +} + +static void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) +{ + int i; + + for (i = 0; i < chs->num; i++) + mlx5e_qos_close_queues(chs->c[i]); +} + +static int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) +{ + u16 qos_sqs_size; + int i; + + qos_sqs_size = DIV_ROUND_UP(mlx5e_qos_max_leaf_nodes(priv->mdev), chs->num); + + for (i = 0; i < chs->num; i++) { + struct mlx5e_txqsq **sqs; + + sqs = kvcalloc(qos_sqs_size, sizeof(struct mlx5e_txqsq *), GFP_KERNEL); + if (!sqs) + goto err_free; + + WRITE_ONCE(chs->c[i]->qos_sqs_size, qos_sqs_size); + smp_wmb(); /* Pairs with mlx5e_napi_poll. */ + rcu_assign_pointer(chs->c[i]->qos_sqs, sqs); + } + + return 0; + +err_free: + while (--i >= 0) { + struct mlx5e_txqsq **sqs; + + sqs = rcu_replace_pointer(chs->c[i]->qos_sqs, NULL, + lockdep_is_held(&priv->state_lock)); + + synchronize_rcu(); /* Sync with NAPI. */ + kvfree(sqs); + } + return -ENOMEM; +} + +int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) +{ + struct mlx5e_qos_node *node = NULL; + int bkt, err; + + if (!priv->htb.maj_id) + return 0; + + err = mlx5e_qos_alloc_queues(priv, chs); + if (err) + return err; + + hash_for_each(priv->htb.qos_tc2node, bkt, node, hnode) { + if (node->qid == MLX5E_QOS_QID_INNER) + continue; + err = mlx5e_open_qos_sq(priv, chs, node); + if (err) { + mlx5e_qos_close_all_queues(chs); + return err; + } + } + + return 0; +} + +void mlx5e_qos_activate_queues(struct mlx5e_priv *priv) +{ + struct mlx5e_qos_node *node = NULL; + int bkt; + + hash_for_each(priv->htb.qos_tc2node, bkt, node, hnode) { + if (node->qid == MLX5E_QOS_QID_INNER) + continue; + mlx5e_activate_qos_sq(priv, node); + } +} + +void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c) +{ + struct mlx5e_params *params = &c->priv->channels.params; + struct mlx5e_txqsq __rcu **qos_sqs; + int i; + + qos_sqs = mlx5e_state_dereference(c->priv, c->qos_sqs); + if (!qos_sqs) + return; + + for (i = 0; i < c->qos_sqs_size; i++) { + u16 qid = params->num_channels * i + c->ix; + struct mlx5e_txqsq *sq; + + sq = mlx5e_state_dereference(c->priv, qos_sqs[i]); + if (!sq) /* Handle the case when the SQ failed to open. */ + continue; + + qos_dbg(c->mdev, "Deactivate QoS SQ qid %u\n", qid); + mlx5e_deactivate_txqsq(sq); + + /* The queue is disabled, no synchronization with datapath is needed. */ + c->priv->txq2sq[mlx5e_qid_from_qos(&c->priv->channels, qid)] = NULL; + } +} + +static void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) +{ + int i; + + for (i = 0; i < chs->num; i++) + mlx5e_qos_deactivate_queues(chs->c[i]); +} + +/* HTB API */ + +int mlx5e_htb_root_add(struct mlx5e_priv *priv, u16 htb_maj_id, u16 htb_defcls, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *root; + bool opened; + int err; + + qos_dbg(priv->mdev, "TC_HTB_CREATE handle %04x:, default :%04x\n", htb_maj_id, htb_defcls); + + if (!mlx5_qos_is_supported(priv->mdev)) { + NL_SET_ERR_MSG_MOD(extack, + "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); + return -EOPNOTSUPP; + } + + opened = test_bit(MLX5E_STATE_OPENED, &priv->state); + if (opened) { + err = mlx5e_qos_alloc_queues(priv, &priv->channels); + if (err) + return err; + } + + root = mlx5e_sw_node_create_root(priv); + if (IS_ERR(root)) { + err = PTR_ERR(root); + goto err_free_queues; + } + + err = mlx5_qos_create_root_node(priv->mdev, &root->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error. Try upgrading firmware."); + goto err_sw_node_delete; + } + + WRITE_ONCE(priv->htb.defcls, htb_defcls); + /* Order maj_id after defcls - pairs with + * mlx5e_select_queue/mlx5e_select_htb_queues. + */ + smp_store_release(&priv->htb.maj_id, htb_maj_id); + + return 0; + +err_sw_node_delete: + mlx5e_sw_node_delete(priv, root); + +err_free_queues: + if (opened) + mlx5e_qos_close_all_queues(&priv->channels); + return err; +} + +int mlx5e_htb_root_del(struct mlx5e_priv *priv) +{ + struct mlx5e_qos_node *root; + int err; + + qos_dbg(priv->mdev, "TC_HTB_DESTROY\n"); + + WRITE_ONCE(priv->htb.maj_id, 0); + synchronize_rcu(); /* Sync with mlx5e_select_htb_queue and TX data path. */ + + root = mlx5e_sw_node_find(priv, MLX5E_HTB_CLASSID_ROOT); + if (!root) { + qos_err(priv->mdev, "Failed to find the root node in the QoS tree\n"); + return -ENOENT; + } + err = mlx5_qos_destroy_node(priv->mdev, root->hw_id); + if (err) + qos_err(priv->mdev, "Failed to destroy root node %u, err = %d\n", + root->hw_id, err); + mlx5e_sw_node_delete(priv, root); + + mlx5e_qos_deactivate_all_queues(&priv->channels); + mlx5e_qos_close_all_queues(&priv->channels); + + return err; +} + +static int mlx5e_htb_convert_rate(struct mlx5e_priv *priv, u64 rate, + struct mlx5e_qos_node *parent, u32 *bw_share) +{ + u64 share = 0; + + while (parent->classid != MLX5E_HTB_CLASSID_ROOT && !parent->max_average_bw) + parent = parent->parent; + + if (parent->max_average_bw) + share = div64_u64(div_u64(rate * 100, BYTES_IN_MBIT), + parent->max_average_bw); + else + share = 101; + + *bw_share = share == 0 ? 1 : share > 100 ? 0 : share; + + qos_dbg(priv->mdev, "Convert: rate %llu, parent ceil %llu -> bw_share %u\n", + rate, (u64)parent->max_average_bw * BYTES_IN_MBIT, *bw_share); + + return 0; +} + +static void mlx5e_htb_convert_ceil(struct mlx5e_priv *priv, u64 ceil, u32 *max_average_bw) +{ + *max_average_bw = div_u64(ceil, BYTES_IN_MBIT); + + qos_dbg(priv->mdev, "Convert: ceil %llu -> max_average_bw %u\n", + ceil, *max_average_bw); +} + +int mlx5e_htb_leaf_alloc_queue(struct mlx5e_priv *priv, u16 classid, + u32 parent_classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node, *parent; + int qid; + int err; + + qos_dbg(priv->mdev, "TC_HTB_LEAF_ALLOC_QUEUE classid %04x, parent %04x, rate %llu, ceil %llu\n", + classid, parent_classid, rate, ceil); + + qid = mlx5e_find_unused_qos_qid(priv); + if (qid < 0) { + NL_SET_ERR_MSG_MOD(extack, "Maximum amount of leaf classes is reached."); + return qid; + } + + parent = mlx5e_sw_node_find(priv, parent_classid); + if (!parent) + return -EINVAL; + + node = mlx5e_sw_node_create_leaf(priv, classid, qid, parent); + if (IS_ERR(node)) + return PTR_ERR(node); + + node->rate = rate; + mlx5e_htb_convert_rate(priv, rate, node->parent, &node->bw_share); + mlx5e_htb_convert_ceil(priv, ceil, &node->max_average_bw); + + err = mlx5_qos_create_leaf_node(priv->mdev, node->parent->hw_id, + node->bw_share, node->max_average_bw, + &node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); + qos_err(priv->mdev, "Failed to create a leaf node (class %04x), err = %d\n", + classid, err); + mlx5e_sw_node_delete(priv, node); + return err; + } + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, node); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", + classid, err); + } else { + mlx5e_activate_qos_sq(priv, node); + } + } + + return mlx5e_qid_from_qos(&priv->channels, node->qid); +} + +int mlx5e_htb_leaf_to_inner(struct mlx5e_priv *priv, u16 classid, u16 child_classid, + u64 rate, u64 ceil, struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node, *child; + int err, tmp_err; + u32 new_hw_id; + u16 qid; + + qos_dbg(priv->mdev, "TC_HTB_LEAF_TO_INNER classid %04x, upcoming child %04x, rate %llu, ceil %llu\n", + classid, child_classid, rate, ceil); + + node = mlx5e_sw_node_find(priv, classid); + if (!node) + return -ENOENT; + + err = mlx5_qos_create_inner_node(priv->mdev, node->parent->hw_id, + node->bw_share, node->max_average_bw, + &new_hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating an inner node."); + qos_err(priv->mdev, "Failed to create an inner node (class %04x), err = %d\n", + classid, err); + return err; + } + + /* Intentionally reuse the qid for the upcoming first child. */ + child = mlx5e_sw_node_create_leaf(priv, child_classid, node->qid, node); + if (IS_ERR(child)) { + err = PTR_ERR(child); + goto err_destroy_hw_node; + } + + child->rate = rate; + mlx5e_htb_convert_rate(priv, rate, node, &child->bw_share); + mlx5e_htb_convert_ceil(priv, ceil, &child->max_average_bw); + + err = mlx5_qos_create_leaf_node(priv->mdev, new_hw_id, child->bw_share, + child->max_average_bw, &child->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); + qos_err(priv->mdev, "Failed to create a leaf node (class %04x), err = %d\n", + classid, err); + goto err_delete_sw_node; + } + + /* No fail point. */ + + qid = node->qid; + /* Pairs with mlx5e_get_txq_by_classid. */ + WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + mlx5e_deactivate_qos_sq(priv, qid); + mlx5e_close_qos_sq(priv, qid); + } + + err = mlx5_qos_destroy_node(priv->mdev, node->hw_id); + if (err) /* Not fatal. */ + qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + node->hw_id = new_hw_id; + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, child); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", + classid, err); + } else { + mlx5e_activate_qos_sq(priv, child); + } + } + + return 0; + +err_delete_sw_node: + child->qid = MLX5E_QOS_QID_INNER; + mlx5e_sw_node_delete(priv, child); + +err_destroy_hw_node: + tmp_err = mlx5_qos_destroy_node(priv->mdev, new_hw_id); + if (tmp_err) /* Not fatal. */ + qos_warn(priv->mdev, "Failed to roll back creation of an inner node %u (class %04x), err = %d\n", + new_hw_id, classid, tmp_err); + return err; +} + +static struct mlx5e_qos_node *mlx5e_sw_node_find_by_qid(struct mlx5e_priv *priv, u16 qid) +{ + struct mlx5e_qos_node *node = NULL; + int bkt; + + hash_for_each(priv->htb.qos_tc2node, bkt, node, hnode) + if (node->qid == qid) + break; + + return node; +} + +static void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq) +{ + qos_dbg(priv->mdev, "Reactivate QoS SQ qid %u\n", qid); + netdev_tx_reset_queue(txq); + netif_tx_start_queue(txq); +} + +static void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) +{ + struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, qid); + struct Qdisc *qdisc = dev_queue->qdisc_sleeping; + + if (!qdisc) + return; + + spin_lock_bh(qdisc_lock(qdisc)); + qdisc_reset(qdisc); + spin_unlock_bh(qdisc_lock(qdisc)); +} + +int mlx5e_htb_leaf_del(struct mlx5e_priv *priv, u16 classid, u16 *old_qid, + u16 *new_qid, struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node; + struct netdev_queue *txq; + u16 qid, moved_qid; + bool opened; + int err; + + qos_dbg(priv->mdev, "TC_HTB_LEAF_DEL classid %04x\n", classid); + + *old_qid = *new_qid = 0; + + node = mlx5e_sw_node_find(priv, classid); + if (!node) + return -ENOENT; + + /* Store qid for reuse. */ + qid = node->qid; + + opened = test_bit(MLX5E_STATE_OPENED, &priv->state); + if (opened) { + txq = netdev_get_tx_queue(priv->netdev, + mlx5e_qid_from_qos(&priv->channels, qid)); + mlx5e_deactivate_qos_sq(priv, qid); + mlx5e_close_qos_sq(priv, qid); + } + + err = mlx5_qos_destroy_node(priv->mdev, node->hw_id); + if (err) /* Not fatal. */ + qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + mlx5e_sw_node_delete(priv, node); + + moved_qid = mlx5e_qos_cur_leaf_nodes(priv); + + if (moved_qid == 0) { + /* The last QoS SQ was just destroyed. */ + if (opened) + mlx5e_reactivate_qos_sq(priv, qid, txq); + return 0; + } + moved_qid--; + + if (moved_qid < qid) { + /* The highest QoS SQ was just destroyed. */ + WARN(moved_qid != qid - 1, "Gaps in queue numeration: destroyed queue %u, the highest queue is %u", + qid, moved_qid); + if (opened) + mlx5e_reactivate_qos_sq(priv, qid, txq); + return 0; + } + + WARN(moved_qid == qid, "Can't move node with qid %u to itself", qid); + qos_dbg(priv->mdev, "Moving QoS SQ %u to %u\n", moved_qid, qid); + + node = mlx5e_sw_node_find_by_qid(priv, moved_qid); + WARN(!node, "Could not find a node with qid %u to move to queue %u", + moved_qid, qid); + + /* Stop traffic to the old queue. */ + WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); + __clear_bit(moved_qid, priv->htb.qos_used_qids); + + if (opened) { + txq = netdev_get_tx_queue(priv->netdev, + mlx5e_qid_from_qos(&priv->channels, moved_qid)); + mlx5e_deactivate_qos_sq(priv, moved_qid); + mlx5e_close_qos_sq(priv, moved_qid); + } + + /* Prevent packets from the old class from getting into the new one. */ + mlx5e_reset_qdisc(priv->netdev, moved_qid); + + __set_bit(qid, priv->htb.qos_used_qids); + WRITE_ONCE(node->qid, qid); + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, node); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x) while moving qid %u to %u, err = %d\n", + node->classid, moved_qid, qid, err); + } else { + mlx5e_activate_qos_sq(priv, node); + } + } + + mlx5e_update_tx_netdev_queues(priv); + if (opened) + mlx5e_reactivate_qos_sq(priv, moved_qid, txq); + + *old_qid = mlx5e_qid_from_qos(&priv->channels, moved_qid); + *new_qid = mlx5e_qid_from_qos(&priv->channels, qid); + return 0; +} + +int mlx5e_htb_leaf_del_last(struct mlx5e_priv *priv, u16 classid, bool force, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node, *parent; + u32 old_hw_id, new_hw_id; + int err, saved_err = 0; + u16 qid; + + qos_dbg(priv->mdev, "TC_HTB_LEAF_DEL_LAST%s classid %04x\n", + force ? "_FORCE" : "", classid); + + node = mlx5e_sw_node_find(priv, classid); + if (!node) + return -ENOENT; + + err = mlx5_qos_create_leaf_node(priv->mdev, node->parent->parent->hw_id, + node->parent->bw_share, + node->parent->max_average_bw, + &new_hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); + qos_err(priv->mdev, "Failed to create a leaf node (class %04x), err = %d\n", + classid, err); + if (!force) + return err; + saved_err = err; + } + + /* Store qid for reuse and prevent clearing the bit. */ + qid = node->qid; + /* Pairs with mlx5e_get_txq_by_classid. */ + WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + mlx5e_deactivate_qos_sq(priv, qid); + mlx5e_close_qos_sq(priv, qid); + } + + /* Prevent packets from the old class from getting into the new one. */ + mlx5e_reset_qdisc(priv->netdev, qid); + + err = mlx5_qos_destroy_node(priv->mdev, node->hw_id); + if (err) /* Not fatal. */ + qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + parent = node->parent; + mlx5e_sw_node_delete(priv, node); + + node = parent; + WRITE_ONCE(node->qid, qid); + + /* Early return on error in force mode. Parent will still be an inner + * node to be deleted by a following delete operation. + */ + if (saved_err) + return saved_err; + + old_hw_id = node->hw_id; + node->hw_id = new_hw_id; + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, node); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", + classid, err); + } else { + mlx5e_activate_qos_sq(priv, node); + } + } + + err = mlx5_qos_destroy_node(priv->mdev, old_hw_id); + if (err) /* Not fatal. */ + qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + return 0; +} + +static int mlx5e_qos_update_children(struct mlx5e_priv *priv, struct mlx5e_qos_node *node, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *child; + int err = 0; + int bkt; + + hash_for_each(priv->htb.qos_tc2node, bkt, child, hnode) { + u32 old_bw_share = child->bw_share; + int err_one; + + if (child->parent != node) + continue; + + mlx5e_htb_convert_rate(priv, child->rate, node, &child->bw_share); + if (child->bw_share == old_bw_share) + continue; + + err_one = mlx5_qos_update_node(priv->mdev, child->hw_id, child->bw_share, + child->max_average_bw, child->hw_id); + if (!err && err_one) { + err = err_one; + + NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a child node."); + qos_err(priv->mdev, "Failed to modify a child node (class %04x), err = %d\n", + node->classid, err); + } + } + + return err; +} + +int mlx5e_htb_node_modify(struct mlx5e_priv *priv, u16 classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack) +{ + u32 bw_share, max_average_bw; + struct mlx5e_qos_node *node; + bool ceil_changed = false; + int err; + + qos_dbg(priv->mdev, "TC_HTB_LEAF_MODIFY classid %04x, rate %llu, ceil %llu\n", + classid, rate, ceil); + + node = mlx5e_sw_node_find(priv, classid); + if (!node) + return -ENOENT; + + node->rate = rate; + mlx5e_htb_convert_rate(priv, rate, node->parent, &bw_share); + mlx5e_htb_convert_ceil(priv, ceil, &max_average_bw); + + err = mlx5_qos_update_node(priv->mdev, node->parent->hw_id, bw_share, + max_average_bw, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a node."); + qos_err(priv->mdev, "Failed to modify a node (class %04x), err = %d\n", + classid, err); + return err; + } + + if (max_average_bw != node->max_average_bw) + ceil_changed = true; + + node->bw_share = bw_share; + node->max_average_bw = max_average_bw; + + if (ceil_changed) + err = mlx5e_qos_update_children(priv, node, extack); + + return err; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h new file mode 100644 index 000000000000..5af7991fcd19 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#ifndef __MLX5E_EN_QOS_H +#define __MLX5E_EN_QOS_H + +#include + +#define MLX5E_QOS_MAX_LEAF_NODES 256 + +struct mlx5e_priv; +struct mlx5e_channels; +struct mlx5e_channel; + +int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev); +int mlx5e_qos_cur_leaf_nodes(struct mlx5e_priv *priv); + +/* TX datapath API */ +int mlx5e_get_txq_by_classid(struct mlx5e_priv *priv, u16 classid); +struct mlx5e_txqsq *mlx5e_get_sq(struct mlx5e_priv *priv, int qid); + +/* SQ lifecycle */ +int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs); +void mlx5e_qos_activate_queues(struct mlx5e_priv *priv); +void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c); +void mlx5e_qos_close_queues(struct mlx5e_channel *c); + +/* HTB API */ +int mlx5e_htb_root_add(struct mlx5e_priv *priv, u16 htb_maj_id, u16 htb_defcls, + struct netlink_ext_ack *extack); +int mlx5e_htb_root_del(struct mlx5e_priv *priv); +int mlx5e_htb_leaf_alloc_queue(struct mlx5e_priv *priv, u16 classid, + u32 parent_classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack); +int mlx5e_htb_leaf_to_inner(struct mlx5e_priv *priv, u16 classid, u16 child_classid, + u64 rate, u64 ceil, struct netlink_ext_ack *extack); +int mlx5e_htb_leaf_del(struct mlx5e_priv *priv, u16 classid, u16 *old_qid, + u16 *new_qid, struct netlink_ext_ack *extack); +int mlx5e_htb_leaf_del_last(struct mlx5e_priv *priv, u16 classid, bool force, + struct netlink_ext_ack *extack); +int mlx5e_htb_node_modify(struct mlx5e_priv *priv, u16 classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack); + +#endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 2d37742a888c..2e5a0696374a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -447,6 +447,17 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv, goto out; } + /* Don't allow changing the number of channels if HTB offload is active, + * because the numeration of the QoS SQs will change, while per-queue + * qdiscs are attached. + */ + if (priv->htb.maj_id) { + err = -EINVAL; + netdev_err(priv->netdev, "%s: HTB offload is active, cannot change the number of channels\n", + __func__); + goto out; + } + new_channels.params = priv->channels.params; new_channels.params.num_channels = count; @@ -1966,6 +1977,16 @@ static int set_pflag_tx_port_ts(struct net_device *netdev, bool enable) if (!MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn)) return -EOPNOTSUPP; + /* Don't allow changing the PTP state if HTB offload is active, because + * the numeration of the QoS SQs will change, while per-queue qdiscs are + * attached. + */ + if (priv->htb.maj_id) { + netdev_err(priv->netdev, "%s: HTB offload is active, cannot change the PTP state\n", + __func__); + return -EINVAL; + } + new_channels.params = priv->channels.params; MLX5E_SET_PFLAG(&new_channels.params, MLX5E_PFLAG_TX_PORT_TS, enable); /* No need to verify SQ stop room as diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index f33c38629886..b9a175982801 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -65,6 +65,7 @@ #include "en/devlink.h" #include "lib/mlx5.h" #include "en/ptp.h" +#include "qos.h" bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) { @@ -1143,7 +1144,6 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, sq->uar_map = mdev->mlx5e_res.bfreg.map; sq->min_inline_mode = params->tx_min_inline_mode; sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); - sq->stats = &c->priv->channel_stats[c->ix].sq[tc]; INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work); if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert)) set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state); @@ -1233,6 +1233,7 @@ static int mlx5e_create_sq(struct mlx5_core_dev *mdev, int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn, struct mlx5e_modify_sq_param *p) { + u64 bitmask = 0; void *in; void *sqc; int inlen; @@ -1248,9 +1249,14 @@ int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn, MLX5_SET(modify_sq_in, in, sq_state, p->curr_state); MLX5_SET(sqc, sqc, state, p->next_state); if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) { - MLX5_SET64(modify_sq_in, in, modify_bitmask, 1); - MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index); + bitmask |= 1; + MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index); } + if (p->qos_update && p->next_state == MLX5_SQC_STATE_RDY) { + bitmask |= 1 << 2; + MLX5_SET(sqc, sqc, qos_queue_group_id, p->qos_queue_group_id); + } + MLX5_SET64(modify_sq_in, in, modify_bitmask, bitmask); err = mlx5_core_modify_sq(mdev, sqn, in); @@ -1267,6 +1273,7 @@ static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn) int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, struct mlx5e_sq_param *param, struct mlx5e_create_sq_param *csp, + u16 qos_queue_group_id, u32 *sqn) { struct mlx5e_modify_sq_param msp = {0}; @@ -1278,6 +1285,10 @@ int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, msp.curr_state = MLX5_SQC_STATE_RST; msp.next_state = MLX5_SQC_STATE_RDY; + if (qos_queue_group_id) { + msp.qos_update = true; + msp.qos_queue_group_id = qos_queue_group_id; + } err = mlx5e_modify_sq(mdev, *sqn, &msp); if (err) mlx5e_destroy_sq(mdev, *sqn); @@ -1288,13 +1299,9 @@ int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, static int mlx5e_set_sq_maxrate(struct net_device *dev, struct mlx5e_txqsq *sq, u32 rate); -static int mlx5e_open_txqsq(struct mlx5e_channel *c, - u32 tisn, - int txq_ix, - struct mlx5e_params *params, - struct mlx5e_sq_param *param, - struct mlx5e_txqsq *sq, - int tc) +int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix, + struct mlx5e_params *params, struct mlx5e_sq_param *param, + struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id, u16 qos_qid) { struct mlx5e_create_sq_param csp = {}; u32 tx_rate; @@ -1304,12 +1311,17 @@ static int mlx5e_open_txqsq(struct mlx5e_channel *c, if (err) return err; + if (qos_queue_group_id) + sq->stats = c->priv->htb.qos_sq_stats[qos_qid]; + else + sq->stats = &c->priv->channel_stats[c->ix].sq[tc]; + csp.tisn = tisn; csp.tis_lst_sz = 1; csp.cqn = sq->cq.mcq.cqn; csp.wq_ctrl = &sq->wq_ctrl; csp.min_inline_mode = sq->min_inline_mode; - err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn); + err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn); if (err) goto err_free_txqsq; @@ -1366,7 +1378,7 @@ void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq) } } -static void mlx5e_close_txqsq(struct mlx5e_txqsq *sq) +void mlx5e_close_txqsq(struct mlx5e_txqsq *sq) { struct mlx5_core_dev *mdev = sq->mdev; struct mlx5_rate_limit rl = {0}; @@ -1403,7 +1415,7 @@ int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params, csp.cqn = sq->cq.mcq.cqn; csp.wq_ctrl = &sq->wq_ctrl; csp.min_inline_mode = params->tx_min_inline_mode; - err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn); + err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); if (err) goto err_free_icosq; @@ -1452,7 +1464,7 @@ int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, csp.wq_ctrl = &sq->wq_ctrl; csp.min_inline_mode = sq->min_inline_mode; set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); - err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn); + err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); if (err) goto err_free_xdpsq; @@ -1703,7 +1715,7 @@ static int mlx5e_open_sqs(struct mlx5e_channel *c, int txq_ix = c->ix + tc * params->num_channels; err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix, - params, &cparam->txq_sq, &c->sq[tc], tc); + params, &cparam->txq_sq, &c->sq[tc], tc, 0, 0); if (err) goto err_close_sqs; } @@ -2044,6 +2056,8 @@ static void mlx5e_deactivate_channel(struct mlx5e_channel *c) mlx5e_deactivate_icosq(&c->icosq); for (tc = 0; tc < c->num_tc; tc++) mlx5e_deactivate_txqsq(&c->sq[tc]); + + mlx5e_qos_deactivate_queues(c); } static void mlx5e_close_channel(struct mlx5e_channel *c) @@ -2051,6 +2065,7 @@ static void mlx5e_close_channel(struct mlx5e_channel *c) if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) mlx5e_close_xsk(c); mlx5e_close_queues(c); + mlx5e_qos_close_queues(c); netif_napi_del(&c->napi); kvfree(c); @@ -2198,9 +2213,8 @@ void mlx5e_build_sq_param_common(struct mlx5e_priv *priv, param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(priv->mdev)); } -static void mlx5e_build_sq_param(struct mlx5e_priv *priv, - struct mlx5e_params *params, - struct mlx5e_sq_param *param) +void mlx5e_build_sq_param(struct mlx5e_priv *priv, struct mlx5e_params *params, + struct mlx5e_sq_param *param) { void *sqc = param->sqc; void *wq = MLX5_ADDR_OF(sqc, sqc, wq); @@ -2379,10 +2393,18 @@ int mlx5e_open_channels(struct mlx5e_priv *priv, goto err_close_channels; } + err = mlx5e_qos_open_queues(priv, chs); + if (err) + goto err_close_ptp; + mlx5e_health_channels_update(priv); kvfree(cparam); return 0; +err_close_ptp: + if (chs->port_ptp) + mlx5e_port_ptp_close(chs->port_ptp); + err_close_channels: for (i--; i >= 0; i--) mlx5e_close_channel(chs->c[i]); @@ -2915,11 +2937,31 @@ static void mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc) netdev_set_tc_queue(netdev, tc, nch, 0); } +int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv) +{ + int qos_queues, nch, ntc, num_txqs, err; + + qos_queues = mlx5e_qos_cur_leaf_nodes(priv); + + nch = priv->channels.params.num_channels; + ntc = priv->channels.params.num_tc; + num_txqs = nch * ntc + qos_queues; + if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS)) + num_txqs += ntc; + + mlx5e_dbg(DRV, priv, "Setting num_txqs %d\n", num_txqs); + err = netif_set_real_num_tx_queues(priv->netdev, num_txqs); + if (err) + netdev_warn(priv->netdev, "netif_set_real_num_tx_queues failed, %d\n", err); + + return err; +} + static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv) { struct net_device *netdev = priv->netdev; - int num_txqs, num_rxqs, nch, ntc; int old_num_txqs, old_ntc; + int num_rxqs, nch, ntc; int err; old_num_txqs = netdev->real_num_tx_queues; @@ -2927,18 +2969,13 @@ static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv) nch = priv->channels.params.num_channels; ntc = priv->channels.params.num_tc; - num_txqs = nch * ntc; - if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS)) - num_txqs += ntc; num_rxqs = nch * priv->profile->rq_groups; mlx5e_netdev_set_tcs(netdev, nch, ntc); - err = netif_set_real_num_tx_queues(netdev, num_txqs); - if (err) { - netdev_warn(netdev, "netif_set_real_num_tx_queues failed, %d\n", err); + err = mlx5e_update_tx_netdev_queues(priv); + if (err) goto err_tcs; - } err = netif_set_real_num_rx_queues(netdev, num_rxqs); if (err) { netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err); @@ -3042,6 +3079,7 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) mlx5e_update_num_tc_x_num_ch(priv); mlx5e_build_txq_maps(priv); mlx5e_activate_channels(&priv->channels); + mlx5e_qos_activate_queues(priv); mlx5e_xdp_tx_enable(priv); netif_tx_start_all_queues(priv->netdev); @@ -3608,6 +3646,14 @@ static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, mutex_lock(&priv->state_lock); + /* MQPRIO is another toplevel qdisc that can't be attached + * simultaneously with the offloaded HTB. + */ + if (WARN_ON(priv->htb.maj_id)) { + err = -EINVAL; + goto out; + } + new_channels.params = priv->channels.params; new_channels.params.num_tc = tc ? tc : 1; @@ -3628,12 +3674,55 @@ out: return err; } +static int mlx5e_setup_tc_htb(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb) +{ + int res; + + switch (htb->command) { + case TC_HTB_CREATE: + return mlx5e_htb_root_add(priv, htb->parent_classid, htb->classid, + htb->extack); + case TC_HTB_DESTROY: + return mlx5e_htb_root_del(priv); + case TC_HTB_LEAF_ALLOC_QUEUE: + res = mlx5e_htb_leaf_alloc_queue(priv, htb->classid, htb->parent_classid, + htb->rate, htb->ceil, htb->extack); + if (res < 0) + return res; + htb->qid = res; + return 0; + case TC_HTB_LEAF_TO_INNER: + return mlx5e_htb_leaf_to_inner(priv, htb->parent_classid, htb->classid, + htb->rate, htb->ceil, htb->extack); + case TC_HTB_LEAF_DEL: + return mlx5e_htb_leaf_del(priv, htb->classid, &htb->moved_qid, &htb->qid, + htb->extack); + case TC_HTB_LEAF_DEL_LAST: + case TC_HTB_LEAF_DEL_LAST_FORCE: + return mlx5e_htb_leaf_del_last(priv, htb->classid, + htb->command == TC_HTB_LEAF_DEL_LAST_FORCE, + htb->extack); + case TC_HTB_NODE_MODIFY: + return mlx5e_htb_node_modify(priv, htb->classid, htb->rate, htb->ceil, + htb->extack); + case TC_HTB_LEAF_QUERY_QUEUE: + res = mlx5e_get_txq_by_classid(priv, htb->classid); + if (res < 0) + return res; + htb->qid = res; + return 0; + default: + return -EOPNOTSUPP; + } +} + static LIST_HEAD(mlx5e_block_cb_list); static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) { struct mlx5e_priv *priv = netdev_priv(dev); + int err; switch (type) { case TC_SETUP_BLOCK: { @@ -3647,6 +3736,11 @@ static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, } case TC_SETUP_QDISC_MQPRIO: return mlx5e_setup_tc_mqprio(priv, type_data); + case TC_SETUP_QDISC_HTB: + mutex_lock(&priv->state_lock); + err = mlx5e_setup_tc_htb(priv, type_data); + mutex_unlock(&priv->state_lock); + return err; default: return -EOPNOTSUPP; } @@ -3811,20 +3905,25 @@ static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) return 0; } -#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) -static int set_feature_tc_num_filters(struct net_device *netdev, bool enable) +static int set_feature_hw_tc(struct net_device *netdev, bool enable) { struct mlx5e_priv *priv = netdev_priv(netdev); +#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) if (!enable && mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD))) { netdev_err(netdev, "Active offloaded tc filters, can't turn hw_tc_offload off\n"); return -EINVAL; } +#endif + + if (!enable && priv->htb.maj_id) { + netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n"); + return -EINVAL; + } return 0; } -#endif static int set_feature_rx_all(struct net_device *netdev, bool enable) { @@ -3922,9 +4021,7 @@ int mlx5e_set_features(struct net_device *netdev, netdev_features_t features) err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER, set_feature_cvlan_filter); -#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) - err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_tc_num_filters); -#endif + err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_hw_tc); err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all); err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan); @@ -5028,6 +5125,8 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) netdev->hw_features |= NETIF_F_NTUPLE; #endif } + if (mlx5_qos_is_supported(mdev)) + netdev->features |= NETIF_F_HW_TC; netdev->features |= NETIF_F_HIGHDMA; netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER; @@ -5333,6 +5432,7 @@ int mlx5e_netdev_init(struct net_device *netdev, return -ENOMEM; mutex_init(&priv->state_lock); + hash_init(priv->htb.qos_tc2node); INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work); @@ -5355,8 +5455,14 @@ err_free_cpumask: void mlx5e_netdev_cleanup(struct net_device *netdev, struct mlx5e_priv *priv) { + int i; + destroy_workqueue(priv->wq); free_cpumask_var(priv->scratchpad.cpumask); + + for (i = 0; i < priv->htb.max_qos_sqs; i++) + kfree(priv->htb.qos_sq_stats[i]); + kvfree(priv->htb.qos_sq_stats); } struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev, @@ -5366,13 +5472,17 @@ struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev, { struct net_device *netdev; unsigned int ptp_txqs = 0; + int qos_sqs = 0; int err; if (MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn)) ptp_txqs = profile->max_tc; + if (mlx5_qos_is_supported(mdev)) + qos_sqs = mlx5e_qos_max_leaf_nodes(mdev); + netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), - nch * profile->max_tc + ptp_txqs, + nch * profile->max_tc + ptp_txqs + qos_sqs, nch * profile->rq_groups); if (!netdev) { mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n"); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 2cf2042b37c7..92c5b81427b9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -420,6 +420,25 @@ static void mlx5e_stats_grp_sw_update_stats_ptp(struct mlx5e_priv *priv, } } +static void mlx5e_stats_grp_sw_update_stats_qos(struct mlx5e_priv *priv, + struct mlx5e_sw_stats *s) +{ + struct mlx5e_sq_stats **stats; + u16 max_qos_sqs; + int i; + + /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ + max_qos_sqs = smp_load_acquire(&priv->htb.max_qos_sqs); + stats = READ_ONCE(priv->htb.qos_sq_stats); + + for (i = 0; i < max_qos_sqs; i++) { + mlx5e_stats_grp_sw_update_stats_sq(s, READ_ONCE(stats[i])); + + /* https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92657 */ + barrier(); + } +} + static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw) { struct mlx5e_sw_stats *s = &priv->stats.sw; @@ -449,6 +468,7 @@ static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw) } } mlx5e_stats_grp_sw_update_stats_ptp(priv, s); + mlx5e_stats_grp_sw_update_stats_qos(priv, s); } static const struct counter_desc q_stats_desc[] = { @@ -1740,6 +1760,41 @@ static const struct counter_desc ptp_cq_stats_desc[] = { { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, abort_abs_diff_ns) }, }; +static const struct counter_desc qos_sq_stats_desc[] = { + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, packets) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, bytes) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tso_packets) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tso_bytes) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tso_inner_packets) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tso_inner_bytes) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, csum_partial) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, csum_partial_inner) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, added_vlan_packets) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, nop) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, mpwqe_blks) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, mpwqe_pkts) }, +#ifdef CONFIG_MLX5_EN_TLS + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tls_encrypted_packets) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tls_encrypted_bytes) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tls_ctx) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tls_ooo) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tls_dump_packets) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tls_dump_bytes) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tls_resync_bytes) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tls_skip_no_sync_data) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tls_drop_no_sync_data) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, tls_drop_bypass_req) }, +#endif + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, csum_none) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, stopped) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, dropped) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, xmit_more) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, recover) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, cqes) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, wake) }, + { MLX5E_DECLARE_QOS_TX_STAT(struct mlx5e_sq_stats, cqe_err) }, +}; + #define NUM_RQ_STATS ARRAY_SIZE(rq_stats_desc) #define NUM_SQ_STATS ARRAY_SIZE(sq_stats_desc) #define NUM_XDPSQ_STATS ARRAY_SIZE(xdpsq_stats_desc) @@ -1750,6 +1805,49 @@ static const struct counter_desc ptp_cq_stats_desc[] = { #define NUM_PTP_SQ_STATS ARRAY_SIZE(ptp_sq_stats_desc) #define NUM_PTP_CH_STATS ARRAY_SIZE(ptp_ch_stats_desc) #define NUM_PTP_CQ_STATS ARRAY_SIZE(ptp_cq_stats_desc) +#define NUM_QOS_SQ_STATS ARRAY_SIZE(qos_sq_stats_desc) + +static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(qos) +{ + /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ + return NUM_QOS_SQ_STATS * smp_load_acquire(&priv->htb.max_qos_sqs); +} + +static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(qos) +{ + /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ + u16 max_qos_sqs = smp_load_acquire(&priv->htb.max_qos_sqs); + int i, qid; + + for (qid = 0; qid < max_qos_sqs; qid++) + for (i = 0; i < NUM_QOS_SQ_STATS; i++) + sprintf(data + (idx++) * ETH_GSTRING_LEN, + qos_sq_stats_desc[i].format, qid); + + return idx; +} + +static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(qos) +{ + struct mlx5e_sq_stats **stats; + u16 max_qos_sqs; + int i, qid; + + /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ + max_qos_sqs = smp_load_acquire(&priv->htb.max_qos_sqs); + stats = READ_ONCE(priv->htb.qos_sq_stats); + + for (qid = 0; qid < max_qos_sqs; qid++) { + struct mlx5e_sq_stats *s = READ_ONCE(stats[qid]); + + for (i = 0; i < NUM_QOS_SQ_STATS; i++) + data[idx++] = MLX5E_READ_CTR64_CPU(s, qos_sq_stats_desc, i); + } + + return idx; +} + +static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(qos) { return; } static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(ptp) { @@ -1932,6 +2030,7 @@ MLX5E_DEFINE_STATS_GRP(per_port_buff_congest, 0); MLX5E_DEFINE_STATS_GRP(eth_ext, 0); static MLX5E_DEFINE_STATS_GRP(tls, 0); static MLX5E_DEFINE_STATS_GRP(ptp, 0); +static MLX5E_DEFINE_STATS_GRP(qos, 0); /* The stats groups order is opposite to the update_stats() order calls */ mlx5e_stats_grp_t mlx5e_nic_stats_grps[] = { @@ -1955,6 +2054,7 @@ mlx5e_stats_grp_t mlx5e_nic_stats_grps[] = { &MLX5E_STATS_GRP(channels), &MLX5E_STATS_GRP(per_port_buff_congest), &MLX5E_STATS_GRP(ptp), + &MLX5E_STATS_GRP(qos), }; unsigned int mlx5e_nic_stats_grps_num(struct mlx5e_priv *priv) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index e41fc11f2ce7..93c41312fb03 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -55,6 +55,8 @@ #define MLX5E_DECLARE_PTP_CH_STAT(type, fld) "ptp_ch_"#fld, offsetof(type, fld) #define MLX5E_DECLARE_PTP_CQ_STAT(type, fld) "ptp_cq%d_"#fld, offsetof(type, fld) +#define MLX5E_DECLARE_QOS_TX_STAT(type, fld) "qos_tx%d_"#fld, offsetof(type, fld) + struct counter_desc { char format[ETH_GSTRING_LEN]; size_t offset; /* Byte offset */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index 74f233eece54..da6a358a8a10 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -106,28 +106,53 @@ return_txq: return priv->port_ptp_tc2realtxq[up]; } +static int mlx5e_select_htb_queue(struct mlx5e_priv *priv, struct sk_buff *skb, + u16 htb_maj_id) +{ + u16 classid; + + if ((TC_H_MAJ(skb->priority) >> 16) == htb_maj_id) + classid = TC_H_MIN(skb->priority); + else + classid = READ_ONCE(priv->htb.defcls); + + if (!classid) + return 0; + + return mlx5e_get_txq_by_classid(priv, classid); +} + u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, struct net_device *sb_dev) { struct mlx5e_priv *priv = netdev_priv(dev); + int num_tc_x_num_ch; int txq_ix; int up = 0; int ch_ix; - if (unlikely(priv->channels.port_ptp)) { - int num_tc_x_num_ch; + /* Sync with mlx5e_update_num_tc_x_num_ch - avoid refetching. */ + num_tc_x_num_ch = READ_ONCE(priv->num_tc_x_num_ch); + if (unlikely(dev->real_num_tx_queues > num_tc_x_num_ch)) { + /* Order maj_id before defcls - pairs with mlx5e_htb_root_add. */ + u16 htb_maj_id = smp_load_acquire(&priv->htb.maj_id); - if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && - mlx5e_use_ptpsq(skb)) - return mlx5e_select_ptpsq(dev, skb); + if (unlikely(htb_maj_id)) { + txq_ix = mlx5e_select_htb_queue(priv, skb, htb_maj_id); + if (txq_ix > 0) + return txq_ix; + } - /* Sync with mlx5e_update_num_tc_x_num_ch - avoid refetching. */ - num_tc_x_num_ch = READ_ONCE(priv->num_tc_x_num_ch); + if (unlikely(priv->channels.port_ptp)) + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && + mlx5e_use_ptpsq(skb)) + return mlx5e_select_ptpsq(dev, skb); txq_ix = netdev_pick_tx(dev, skb, NULL); - /* Fix netdev_pick_tx() not to choose ptp_channel txqs. + /* Fix netdev_pick_tx() not to choose ptp_channel and HTB txqs. * If they are selected, switch to regular queues. - * Driver to select these queues only at mlx5e_select_ptpsq(). + * Driver to select these queues only at mlx5e_select_ptpsq() + * and mlx5e_select_htb_queue(). */ if (unlikely(txq_ix >= num_tc_x_num_ch)) txq_ix %= num_tc_x_num_ch; @@ -702,6 +727,10 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) u16 pi; sq = priv->txq2sq[skb_get_queue_mapping(skb)]; + if (unlikely(!sq)) { + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } /* May send SKBs and WQEs. */ if (unlikely(!mlx5e_accel_tx_begin(dev, sq, skb, &accel))) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c index a3cfe06d5116..d54da3797c30 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c @@ -115,17 +115,21 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) napi); struct mlx5e_ch_stats *ch_stats = c->stats; struct mlx5e_xdpsq *xsksq = &c->xsksq; + struct mlx5e_txqsq __rcu **qos_sqs; struct mlx5e_rq *xskrq = &c->xskrq; struct mlx5e_rq *rq = &c->rq; bool aff_change = false; bool busy_xsk = false; bool busy = false; int work_done = 0; + u16 qos_sqs_size; bool xsk_open; int i; rcu_read_lock(); + qos_sqs = rcu_dereference(c->qos_sqs); + xsk_open = test_bit(MLX5E_CHANNEL_STATE_XSK, c->state); ch_stats->poll++; @@ -133,6 +137,18 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) for (i = 0; i < c->num_tc; i++) busy |= mlx5e_poll_tx_cq(&c->sq[i].cq, budget); + if (unlikely(qos_sqs)) { + smp_rmb(); /* Pairs with mlx5e_qos_alloc_queues. */ + qos_sqs_size = READ_ONCE(c->qos_sqs_size); + + for (i = 0; i < qos_sqs_size; i++) { + struct mlx5e_txqsq *sq = rcu_dereference(qos_sqs[i]); + + if (sq) + busy |= mlx5e_poll_tx_cq(&sq->cq, budget); + } + } + busy |= mlx5e_poll_xdpsq_cq(&c->xdpsq.cq); if (c->xdp) @@ -186,6 +202,16 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) mlx5e_handle_tx_dim(&c->sq[i]); mlx5e_cq_arm(&c->sq[i].cq); } + if (unlikely(qos_sqs)) { + for (i = 0; i < qos_sqs_size; i++) { + struct mlx5e_txqsq *sq = rcu_dereference(qos_sqs[i]); + + if (sq) { + mlx5e_handle_tx_dim(sq); + mlx5e_cq_arm(&sq->cq); + } + } + } mlx5e_handle_rx_dim(rq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/qos.c new file mode 100644 index 000000000000..0777be24a307 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/qos.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#include "qos.h" + +#define MLX5_QOS_DEFAULT_DWRR_UID 0 + +bool mlx5_qos_is_supported(struct mlx5_core_dev *mdev) +{ + if (!MLX5_CAP_GEN(mdev, qos)) + return false; + if (!MLX5_CAP_QOS(mdev, nic_sq_scheduling)) + return false; + if (!MLX5_CAP_QOS(mdev, nic_bw_share)) + return false; + if (!MLX5_CAP_QOS(mdev, nic_rate_limit)) + return false; + return true; +} + +int mlx5_qos_max_leaf_nodes(struct mlx5_core_dev *mdev) +{ + return 1 << MLX5_CAP_QOS(mdev, log_max_qos_nic_queue_group); +} + +int mlx5_qos_create_leaf_node(struct mlx5_core_dev *mdev, u32 parent_id, + u32 bw_share, u32 max_avg_bw, u32 *id) +{ + u32 sched_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {0}; + + MLX5_SET(scheduling_context, sched_ctx, parent_element_id, parent_id); + MLX5_SET(scheduling_context, sched_ctx, element_type, + SCHEDULING_CONTEXT_ELEMENT_TYPE_QUEUE_GROUP); + MLX5_SET(scheduling_context, sched_ctx, bw_share, bw_share); + MLX5_SET(scheduling_context, sched_ctx, max_average_bw, max_avg_bw); + + return mlx5_create_scheduling_element_cmd(mdev, SCHEDULING_HIERARCHY_NIC, + sched_ctx, id); +} + +int mlx5_qos_create_inner_node(struct mlx5_core_dev *mdev, u32 parent_id, + u32 bw_share, u32 max_avg_bw, u32 *id) +{ + u32 sched_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {0}; + void *attr; + + MLX5_SET(scheduling_context, sched_ctx, parent_element_id, parent_id); + MLX5_SET(scheduling_context, sched_ctx, element_type, + SCHEDULING_CONTEXT_ELEMENT_TYPE_TSAR); + MLX5_SET(scheduling_context, sched_ctx, bw_share, bw_share); + MLX5_SET(scheduling_context, sched_ctx, max_average_bw, max_avg_bw); + + attr = MLX5_ADDR_OF(scheduling_context, sched_ctx, element_attributes); + MLX5_SET(tsar_element, attr, tsar_type, TSAR_ELEMENT_TSAR_TYPE_DWRR); + + return mlx5_create_scheduling_element_cmd(mdev, SCHEDULING_HIERARCHY_NIC, + sched_ctx, id); +} + +int mlx5_qos_create_root_node(struct mlx5_core_dev *mdev, u32 *id) +{ + return mlx5_qos_create_inner_node(mdev, MLX5_QOS_DEFAULT_DWRR_UID, 0, 0, id); +} + +int mlx5_qos_update_node(struct mlx5_core_dev *mdev, u32 parent_id, + u32 bw_share, u32 max_avg_bw, u32 id) +{ + u32 sched_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {0}; + u32 bitmask = 0; + + MLX5_SET(scheduling_context, sched_ctx, parent_element_id, parent_id); + MLX5_SET(scheduling_context, sched_ctx, bw_share, bw_share); + MLX5_SET(scheduling_context, sched_ctx, max_average_bw, max_avg_bw); + + bitmask |= MODIFY_SCHEDULING_ELEMENT_IN_MODIFY_BITMASK_BW_SHARE; + bitmask |= MODIFY_SCHEDULING_ELEMENT_IN_MODIFY_BITMASK_MAX_AVERAGE_BW; + + return mlx5_modify_scheduling_element_cmd(mdev, SCHEDULING_HIERARCHY_NIC, + sched_ctx, id, bitmask); +} + +int mlx5_qos_destroy_node(struct mlx5_core_dev *mdev, u32 id) +{ + return mlx5_destroy_scheduling_element_cmd(mdev, SCHEDULING_HIERARCHY_NIC, id); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/qos.h b/drivers/net/ethernet/mellanox/mlx5/core/qos.h new file mode 100644 index 000000000000..125e4e47e6f7 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/qos.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ + +#ifndef __MLX5_QOS_H +#define __MLX5_QOS_H + +#include "mlx5_core.h" + +#define MLX5_DEBUG_QOS_MASK BIT(4) + +#define qos_err(mdev, fmt, ...) \ + mlx5_core_err(mdev, "QoS: " fmt, ##__VA_ARGS__) +#define qos_warn(mdev, fmt, ...) \ + mlx5_core_warn(mdev, "QoS: " fmt, ##__VA_ARGS__) +#define qos_dbg(mdev, fmt, ...) \ + mlx5_core_dbg_mask(mdev, MLX5_DEBUG_QOS_MASK, "QoS: " fmt, ##__VA_ARGS__) + +bool mlx5_qos_is_supported(struct mlx5_core_dev *mdev); +int mlx5_qos_max_leaf_nodes(struct mlx5_core_dev *mdev); + +int mlx5_qos_create_leaf_node(struct mlx5_core_dev *mdev, u32 parent_id, + u32 bw_share, u32 max_avg_bw, u32 *id); +int mlx5_qos_create_inner_node(struct mlx5_core_dev *mdev, u32 parent_id, + u32 bw_share, u32 max_avg_bw, u32 *id); +int mlx5_qos_create_root_node(struct mlx5_core_dev *mdev, u32 *id); +int mlx5_qos_update_node(struct mlx5_core_dev *mdev, u32 parent_id, u32 bw_share, + u32 max_avg_bw, u32 id); +int mlx5_qos_destroy_node(struct mlx5_core_dev *mdev, u32 id); + +#endif diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 823411e288c0..71ae6aac3410 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -842,11 +842,16 @@ struct mlx5_ifc_qos_cap_bits { u8 reserved_at_4[0x1]; u8 packet_pacing_burst_bound[0x1]; u8 packet_pacing_typical_size[0x1]; - u8 reserved_at_7[0x4]; + u8 reserved_at_7[0x1]; + u8 nic_sq_scheduling[0x1]; + u8 nic_bw_share[0x1]; + u8 nic_rate_limit[0x1]; u8 packet_pacing_uid[0x1]; u8 reserved_at_c[0x14]; - u8 reserved_at_20[0x20]; + u8 reserved_at_20[0xb]; + u8 log_max_qos_nic_queue_group[0x5]; + u8 reserved_at_30[0x10]; u8 packet_pacing_max_rate[0x20]; @@ -3347,7 +3352,7 @@ struct mlx5_ifc_sqc_bits { u8 reserved_at_e0[0x10]; u8 packet_pacing_rate_limit_index[0x10]; u8 tis_lst_sz[0x10]; - u8 reserved_at_110[0x10]; + u8 qos_queue_group_id[0x10]; u8 reserved_at_120[0x40]; @@ -3362,6 +3367,7 @@ enum { SCHEDULING_CONTEXT_ELEMENT_TYPE_VPORT = 0x1, SCHEDULING_CONTEXT_ELEMENT_TYPE_VPORT_TC = 0x2, SCHEDULING_CONTEXT_ELEMENT_TYPE_PARA_VPORT_TC = 0x3, + SCHEDULING_CONTEXT_ELEMENT_TYPE_QUEUE_GROUP = 0x4, }; enum { @@ -4805,6 +4811,7 @@ struct mlx5_ifc_query_scheduling_element_out_bits { enum { SCHEDULING_HIERARCHY_E_SWITCH = 0x2, + SCHEDULING_HIERARCHY_NIC = 0x3, }; struct mlx5_ifc_query_scheduling_element_in_bits { -- cgit v1.2.3 From 2f63296578cad1ae681152d5b2122a4595195f16 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Jan 2021 10:06:09 -0800 Subject: iomap: pass a flags argument to iomap_dio_rw Pass a set of flags to iomap_dio_rw instead of the boolean wait_for_completion argument. The IOMAP_DIO_FORCE_WAIT flag replaces the wait_for_completion, but only needs to be passed when the iocb isn't synchronous to start with to simplify the callers. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: Brian Foster [djwong: rework xfs_file.c so that we can push iomap changes separately] Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/btrfs/file.c | 7 +++---- fs/ext4/file.c | 5 ++--- fs/gfs2/file.c | 7 ++----- fs/iomap/direct-io.c | 11 +++++------ fs/xfs/xfs_file.c | 5 ++--- fs/zonefs/super.c | 4 ++-- include/linux/iomap.h | 10 ++++++++-- 7 files changed, 24 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 0e41459b8de6..ddfd2e2adedf 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -1949,8 +1949,8 @@ relock: goto buffered; } - dio = __iomap_dio_rw(iocb, from, &btrfs_dio_iomap_ops, - &btrfs_dio_ops, is_sync_kiocb(iocb)); + dio = __iomap_dio_rw(iocb, from, &btrfs_dio_iomap_ops, &btrfs_dio_ops, + 0); btrfs_inode_unlock(inode, ilock_flags); @@ -3622,8 +3622,7 @@ static ssize_t btrfs_direct_read(struct kiocb *iocb, struct iov_iter *to) return 0; btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED); - ret = iomap_dio_rw(iocb, to, &btrfs_dio_iomap_ops, &btrfs_dio_ops, - is_sync_kiocb(iocb)); + ret = iomap_dio_rw(iocb, to, &btrfs_dio_iomap_ops, &btrfs_dio_ops, 0); btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); return ret; } diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 349b27f0dda0..194f5d00fa32 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -74,8 +74,7 @@ static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) return generic_file_read_iter(iocb, to); } - ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, - is_sync_kiocb(iocb)); + ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0); inode_unlock_shared(inode); file_accessed(iocb->ki_filp); @@ -550,7 +549,7 @@ static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) if (ilock_shared) iomap_ops = &ext4_iomap_overwrite_ops; ret = iomap_dio_rw(iocb, from, iomap_ops, &ext4_dio_write_ops, - is_sync_kiocb(iocb) || unaligned_io || extend); + (unaligned_io || extend) ? IOMAP_DIO_FORCE_WAIT : 0); if (ret == -ENOTBLK) ret = 0; diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c index b39b339feddc..89609c299717 100644 --- a/fs/gfs2/file.c +++ b/fs/gfs2/file.c @@ -797,9 +797,7 @@ static ssize_t gfs2_file_direct_read(struct kiocb *iocb, struct iov_iter *to, if (ret) goto out_uninit; - ret = iomap_dio_rw(iocb, to, &gfs2_iomap_ops, NULL, - is_sync_kiocb(iocb)); - + ret = iomap_dio_rw(iocb, to, &gfs2_iomap_ops, NULL, 0); gfs2_glock_dq(gh); out_uninit: gfs2_holder_uninit(gh); @@ -833,8 +831,7 @@ static ssize_t gfs2_file_direct_write(struct kiocb *iocb, struct iov_iter *from, if (offset + len > i_size_read(&ip->i_inode)) goto out; - ret = iomap_dio_rw(iocb, from, &gfs2_iomap_ops, NULL, - is_sync_kiocb(iocb)); + ret = iomap_dio_rw(iocb, from, &gfs2_iomap_ops, NULL, 0); if (ret == -ENOTBLK) ret = 0; out: diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 604103ab76f9..947343730e2c 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -420,13 +420,15 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length, struct iomap_dio * __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, - bool wait_for_completion) + unsigned int dio_flags) { struct address_space *mapping = iocb->ki_filp->f_mapping; struct inode *inode = file_inode(iocb->ki_filp); size_t count = iov_iter_count(iter); loff_t pos = iocb->ki_pos; loff_t end = iocb->ki_pos + count - 1, ret = 0; + bool wait_for_completion = + is_sync_kiocb(iocb) || (dio_flags & IOMAP_DIO_FORCE_WAIT); unsigned int iomap_flags = IOMAP_DIRECT; struct blk_plug plug; struct iomap_dio *dio; @@ -434,9 +436,6 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, if (!count) return NULL; - if (WARN_ON(is_sync_kiocb(iocb) && !wait_for_completion)) - return ERR_PTR(-EIO); - dio = kmalloc(sizeof(*dio), GFP_KERNEL); if (!dio) return ERR_PTR(-ENOMEM); @@ -598,11 +597,11 @@ EXPORT_SYMBOL_GPL(__iomap_dio_rw); ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, - bool wait_for_completion) + unsigned int dio_flags) { struct iomap_dio *dio; - dio = __iomap_dio_rw(iocb, iter, ops, dops, wait_for_completion); + dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags); if (IS_ERR_OR_NULL(dio)) return PTR_ERR_OR_ZERO(dio); return iomap_dio_complete(dio); diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 5b0f93f73837..7f18dae84584 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -219,8 +219,7 @@ xfs_file_dio_aio_read( } else { xfs_ilock(ip, XFS_IOLOCK_SHARED); } - ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, - is_sync_kiocb(iocb)); + ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, 0); xfs_iunlock(ip, XFS_IOLOCK_SHARED); return ret; @@ -584,7 +583,7 @@ xfs_file_dio_aio_write( */ ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops, &xfs_dio_write_ops, - is_sync_kiocb(iocb) || unaligned_io); + unaligned_io ? IOMAP_DIO_FORCE_WAIT : 0); out: xfs_iunlock(ip, iolock); diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index bec47f2d074b..0e7ab0bc00ae 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -780,7 +780,7 @@ static ssize_t zonefs_file_dio_write(struct kiocb *iocb, struct iov_iter *from) ret = zonefs_file_dio_append(iocb, from); else ret = iomap_dio_rw(iocb, from, &zonefs_iomap_ops, - &zonefs_write_dio_ops, sync); + &zonefs_write_dio_ops, 0); if (zi->i_ztype == ZONEFS_ZTYPE_SEQ && (ret > 0 || ret == -EIOCBQUEUED)) { if (ret > 0) @@ -917,7 +917,7 @@ static ssize_t zonefs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) } file_accessed(iocb->ki_filp); ret = iomap_dio_rw(iocb, to, &zonefs_iomap_ops, - &zonefs_read_dio_ops, is_sync_kiocb(iocb)); + &zonefs_read_dio_ops, 0); } else { ret = generic_file_read_iter(iocb, to); if (ret == -EIO) diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 5bd3cac4df9c..be4e1e1e01e8 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -256,12 +256,18 @@ struct iomap_dio_ops { struct bio *bio, loff_t file_offset); }; +/* + * Wait for the I/O to complete in iomap_dio_rw even if the kiocb is not + * synchronous. + */ +#define IOMAP_DIO_FORCE_WAIT (1 << 0) + ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, - bool wait_for_completion); + unsigned int dio_flags); struct iomap_dio *__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, - bool wait_for_completion); + unsigned int dio_flags); ssize_t iomap_dio_complete(struct iomap_dio *dio); int iomap_dio_iopoll(struct kiocb *kiocb, bool spin); -- cgit v1.2.3 From 213f627104daf8589aad8ee73fcaeb603ab0af15 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Jan 2021 10:06:10 -0800 Subject: iomap: add a IOMAP_DIO_OVERWRITE_ONLY flag Add a flag to signal that only pure overwrites are allowed. Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/iomap/direct-io.c | 7 +++++++ include/linux/iomap.h | 8 ++++++++ 2 files changed, 15 insertions(+) (limited to 'include/linux') diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 947343730e2c..65d32364345d 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -485,6 +485,13 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, iomap_flags |= IOMAP_NOWAIT; } + if (dio_flags & IOMAP_DIO_OVERWRITE_ONLY) { + ret = -EAGAIN; + if (pos >= dio->i_size || pos + count > dio->i_size) + goto out_free_dio; + iomap_flags |= IOMAP_OVERWRITE_ONLY; + } + ret = filemap_write_and_wait_range(mapping, pos, end); if (ret) goto out_free_dio; diff --git a/include/linux/iomap.h b/include/linux/iomap.h index be4e1e1e01e8..1a86f520de56 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -122,6 +122,7 @@ struct iomap_page_ops { #define IOMAP_FAULT (1 << 3) /* mapping for page fault */ #define IOMAP_DIRECT (1 << 4) /* direct I/O */ #define IOMAP_NOWAIT (1 << 5) /* do not block */ +#define IOMAP_OVERWRITE_ONLY (1 << 6) /* only pure overwrites allowed */ struct iomap_ops { /* @@ -262,6 +263,13 @@ struct iomap_dio_ops { */ #define IOMAP_DIO_FORCE_WAIT (1 << 0) +/* + * Do not allocate blocks or zero partial blocks, but instead fall back to + * the caller by returning -EAGAIN. Used to optimize direct I/O writes that + * are not aligned to the file system block size. + */ +#define IOMAP_DIO_OVERWRITE_ONLY (1 << 1) + ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, const struct iomap_ops *ops, const struct iomap_dio_ops *dops, unsigned int dio_flags); -- cgit v1.2.3 From 6f1c0ea133a6e4a193a7b285efe209664caeea43 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Fri, 22 Jan 2021 18:19:48 +0000 Subject: net: introduce a netdev feature for UDP GRO forwarding Introduce a new netdev feature, NETIF_F_GRO_UDP_FWD, to allow user to turn UDP GRO on and off for forwarding. Defaults to off to not change current datapath. Suggested-by: Paolo Abeni Signed-off-by: Alexander Lobakin Signed-off-by: Jakub Kicinski --- include/linux/netdev_features.h | 4 +++- net/ethtool/common.c | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h index 934de56644e7..c06d6aaba9df 100644 --- a/include/linux/netdev_features.h +++ b/include/linux/netdev_features.h @@ -84,6 +84,7 @@ enum { NETIF_F_GRO_FRAGLIST_BIT, /* Fraglist GRO */ NETIF_F_HW_MACSEC_BIT, /* Offload MACsec operations */ + NETIF_F_GRO_UDP_FWD_BIT, /* Allow UDP GRO for forwarding */ /* * Add your fresh new feature above and remember to update @@ -157,6 +158,7 @@ enum { #define NETIF_F_GRO_FRAGLIST __NETIF_F(GRO_FRAGLIST) #define NETIF_F_GSO_FRAGLIST __NETIF_F(GSO_FRAGLIST) #define NETIF_F_HW_MACSEC __NETIF_F(HW_MACSEC) +#define NETIF_F_GRO_UDP_FWD __NETIF_F(GRO_UDP_FWD) /* Finds the next feature with the highest number of the range of start till 0. */ @@ -234,7 +236,7 @@ static inline int find_next_netdev_feature(u64 feature, unsigned long start) #define NETIF_F_SOFT_FEATURES (NETIF_F_GSO | NETIF_F_GRO) /* Changeable features with no special hardware requirements that defaults to off. */ -#define NETIF_F_SOFT_FEATURES_OFF NETIF_F_GRO_FRAGLIST +#define NETIF_F_SOFT_FEATURES_OFF (NETIF_F_GRO_FRAGLIST | NETIF_F_GRO_UDP_FWD) #define NETIF_F_VLAN_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \ NETIF_F_HW_VLAN_CTAG_RX | \ diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 24036e3055a1..181220101a6e 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -68,6 +68,7 @@ const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN] = { [NETIF_F_HW_TLS_RX_BIT] = "tls-hw-rx-offload", [NETIF_F_GRO_FRAGLIST_BIT] = "rx-gro-list", [NETIF_F_HW_MACSEC_BIT] = "macsec-hw-offload", + [NETIF_F_GRO_UDP_FWD_BIT] = "rx-udp-gro-forwarding", }; const char -- cgit v1.2.3 From 52f019d43c229afd65dc11c8c1b05b6436bf6765 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 9 Jan 2021 11:42:51 +0100 Subject: block: add a hard-readonly flag to struct gendisk Commit 20bd1d026aac ("scsi: sd: Keep disk read-only when re-reading partition") addressed a long-standing problem with user read-only policy being overridden as a result of a device-initiated revalidate. The commit has since been reverted due to a regression that left some USB devices read-only indefinitely. To fix the underlying problems with revalidate we need to keep track of hardware state and user policy separately. The gendisk has been updated to reflect the current hardware state set by the device driver. This is done to allow returning the device to the hardware state once the user clears the BLKROSET flag. The resulting semantics are as follows: - If BLKROSET sets a given partition read-only, that partition will remain read-only even if the underlying storage stack initiates a revalidate. However, the BLKRRPART ioctl will cause the partition table to be dropped and any user policy on partitions will be lost. - If BLKROSET has not been set, both the whole disk device and any partitions will reflect the current write-protect state of the underlying device. Based on a patch from Martin K. Petersen . Reported-by: Oleksii Kurochko Bugzilla: https://bugzilla.kernel.org/show_bug.cgi?id=201221 Signed-off-by: Christoph Hellwig Reviewed-by: Ming Lei Reviewed-by: Martin K. Petersen Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk-core.c | 4 +--- block/genhd.c | 33 +++++++++++++++++++-------------- block/partitions/core.c | 3 +-- include/linux/genhd.h | 6 ++++-- 4 files changed, 25 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/block/blk-core.c b/block/blk-core.c index 7663a9b94b80..08ff8ca32529 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -694,9 +694,7 @@ static inline bool should_fail_request(struct block_device *part, static inline bool bio_check_ro(struct bio *bio, struct block_device *part) { - const int op = bio_op(bio); - - if (part->bd_read_only && op_is_write(op)) { + if (op_is_write(bio_op(bio)) && bdev_read_only(part)) { char b[BDEVNAME_SIZE]; if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) diff --git a/block/genhd.c b/block/genhd.c index 484a474648d5..1873e4571328 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -1637,27 +1637,32 @@ static void set_disk_ro_uevent(struct gendisk *gd, int ro) kobject_uevent_env(&disk_to_dev(gd)->kobj, KOBJ_CHANGE, envp); } -void set_disk_ro(struct gendisk *disk, int flag) +/** + * set_disk_ro - set a gendisk read-only + * @disk: gendisk to operate on + * @ready_only: %true to set the disk read-only, %false set the disk read/write + * + * This function is used to indicate whether a given disk device should have its + * read-only flag set. set_disk_ro() is typically used by device drivers to + * indicate whether the underlying physical device is write-protected. + */ +void set_disk_ro(struct gendisk *disk, bool read_only) { - struct disk_part_iter piter; - struct block_device *part; - - if (disk->part0->bd_read_only != flag) { - set_disk_ro_uevent(disk, flag); - disk->part0->bd_read_only = flag; + if (read_only) { + if (test_and_set_bit(GD_READ_ONLY, &disk->state)) + return; + } else { + if (!test_and_clear_bit(GD_READ_ONLY, &disk->state)) + return; } - - disk_part_iter_init(&piter, disk, DISK_PITER_INCL_EMPTY); - while ((part = disk_part_iter_next(&piter))) - part->bd_read_only = flag; - disk_part_iter_exit(&piter); + set_disk_ro_uevent(disk, read_only); } - EXPORT_SYMBOL(set_disk_ro); int bdev_read_only(struct block_device *bdev) { - return bdev->bd_read_only; + return bdev->bd_read_only || + test_bit(GD_READ_ONLY, &bdev->bd_disk->state); } EXPORT_SYMBOL(bdev_read_only); diff --git a/block/partitions/core.c b/block/partitions/core.c index e7d776db803b..168d5906077c 100644 --- a/block/partitions/core.c +++ b/block/partitions/core.c @@ -195,7 +195,7 @@ static ssize_t part_start_show(struct device *dev, static ssize_t part_ro_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%d\n", dev_to_bdev(dev)->bd_read_only); + return sprintf(buf, "%d\n", bdev_read_only(dev_to_bdev(dev))); } static ssize_t part_alignment_offset_show(struct device *dev, @@ -361,7 +361,6 @@ static struct block_device *add_partition(struct gendisk *disk, int partno, bdev->bd_start_sect = start; bdev_set_nr_sectors(bdev, len); - bdev->bd_read_only = get_disk_ro(disk); if (info) { err = -ENOMEM; diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 809aaa32d53c..a62ccbfac54b 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -163,6 +163,7 @@ struct gendisk { int flags; unsigned long state; #define GD_NEED_PART_SCAN 0 +#define GD_READ_ONLY 1 struct kobject *slave_dir; struct timer_rand_state *random; @@ -249,11 +250,12 @@ static inline void add_disk_no_queue_reg(struct gendisk *disk) extern void del_gendisk(struct gendisk *gp); extern struct block_device *bdget_disk(struct gendisk *disk, int partno); -extern void set_disk_ro(struct gendisk *disk, int flag); +void set_disk_ro(struct gendisk *disk, bool read_only); static inline int get_disk_ro(struct gendisk *disk) { - return disk->part0->bd_read_only; + return disk->part0->bd_read_only || + test_bit(GD_READ_ONLY, &disk->state); } extern void disk_block_events(struct gendisk *disk); -- cgit v1.2.3 From 309dca309fc39a9e3c31b916393b74bd174fd74e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Jan 2021 11:02:34 +0100 Subject: block: store a block_device pointer in struct bio Replace the gendisk pointer in struct bio with a pointer to the newly improved struct block device. From that the gendisk can be trivially accessed with an extra indirection, but it also allows to directly look up all information related to partition remapping. Signed-off-by: Christoph Hellwig Acked-by: Tejun Heo Signed-off-by: Jens Axboe --- arch/m68k/emu/nfblock.c | 2 +- arch/xtensa/platforms/iss/simdisk.c | 2 +- block/bio-integrity.c | 18 +++++++++--------- block/bio.c | 31 +++++++++++-------------------- block/blk-cgroup.c | 7 ++++--- block/blk-core.c | 37 +++++++++++++++++-------------------- block/blk-crypto-fallback.c | 2 +- block/blk-crypto.c | 2 +- block/blk-merge.c | 17 ++++++++--------- block/blk-mq.c | 2 +- block/blk-throttle.c | 2 +- block/blk.h | 2 -- block/bounce.c | 2 +- block/genhd.c | 2 +- drivers/block/brd.c | 2 +- drivers/block/drbd/drbd_int.h | 4 ++-- drivers/block/drbd/drbd_req.c | 2 +- drivers/block/null_blk/main.c | 2 +- drivers/block/pktcdvd.c | 4 ++-- drivers/block/ps3vram.c | 2 +- drivers/block/rsxx/dev.c | 2 +- drivers/block/umem.c | 2 +- drivers/block/zram/zram_drv.c | 2 +- drivers/lightnvm/pblk-init.c | 2 +- drivers/md/bcache/debug.c | 2 +- drivers/md/bcache/request.c | 7 ++++--- drivers/md/dm-bio-record.h | 9 +++------ drivers/md/dm-raid1.c | 10 +++++----- drivers/md/dm.c | 14 +++++++------- drivers/md/md-linear.c | 2 +- drivers/md/md.c | 2 +- drivers/md/md.h | 6 +++--- drivers/md/raid1.c | 6 +++--- drivers/md/raid10.c | 12 ++++++------ drivers/md/raid5.c | 2 +- drivers/nvdimm/blk.c | 4 ++-- drivers/nvdimm/btt.c | 4 ++-- drivers/nvdimm/pmem.c | 4 ++-- drivers/nvme/host/core.c | 6 +++--- drivers/nvme/host/lightnvm.c | 3 +-- drivers/nvme/host/multipath.c | 6 +++--- drivers/nvme/host/rdma.c | 2 +- drivers/s390/block/dcssblk.c | 2 +- drivers/s390/block/xpram.c | 2 +- fs/btrfs/check-integrity.c | 10 +++++----- fs/btrfs/raid56.c | 7 ++----- fs/btrfs/scrub.c | 2 +- fs/direct-io.c | 2 +- fs/f2fs/data.c | 12 +----------- include/linux/bio.h | 18 ++++++++---------- include/linux/blk-mq.h | 4 ++-- include/linux/blk_types.h | 3 +-- include/linux/blkdev.h | 5 +++-- kernel/trace/blktrace.c | 16 +++++++++------- mm/page_io.c | 2 +- 55 files changed, 154 insertions(+), 184 deletions(-) (limited to 'include/linux') diff --git a/arch/m68k/emu/nfblock.c b/arch/m68k/emu/nfblock.c index 92d26c812441..ba808543161a 100644 --- a/arch/m68k/emu/nfblock.c +++ b/arch/m68k/emu/nfblock.c @@ -61,7 +61,7 @@ struct nfhd_device { static blk_qc_t nfhd_submit_bio(struct bio *bio) { - struct nfhd_device *dev = bio->bi_disk->private_data; + struct nfhd_device *dev = bio->bi_bdev->bd_disk->private_data; struct bio_vec bvec; struct bvec_iter iter; int dir, len, shift; diff --git a/arch/xtensa/platforms/iss/simdisk.c b/arch/xtensa/platforms/iss/simdisk.c index 3447556d276d..fc09be7b1347 100644 --- a/arch/xtensa/platforms/iss/simdisk.c +++ b/arch/xtensa/platforms/iss/simdisk.c @@ -103,7 +103,7 @@ static void simdisk_transfer(struct simdisk *dev, unsigned long sector, static blk_qc_t simdisk_submit_bio(struct bio *bio) { - struct simdisk *dev = bio->bi_disk->private_data; + struct simdisk *dev = bio->bi_bdev->bd_disk->private_data; struct bio_vec bvec; struct bvec_iter iter; sector_t sector = bio->bi_iter.bi_sector; diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 9ffd7e289554..c3e5abcfdc98 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -140,7 +140,7 @@ int bio_integrity_add_page(struct bio *bio, struct page *page, iv = bip->bip_vec + bip->bip_vcnt; if (bip->bip_vcnt && - bvec_gap_to_prev(bio->bi_disk->queue, + bvec_gap_to_prev(bio->bi_bdev->bd_disk->queue, &bip->bip_vec[bip->bip_vcnt - 1], offset)) return 0; @@ -162,7 +162,7 @@ EXPORT_SYMBOL(bio_integrity_add_page); static blk_status_t bio_integrity_process(struct bio *bio, struct bvec_iter *proc_iter, integrity_processing_fn *proc_fn) { - struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); struct blk_integrity_iter iter; struct bvec_iter bviter; struct bio_vec bv; @@ -171,7 +171,7 @@ static blk_status_t bio_integrity_process(struct bio *bio, void *prot_buf = page_address(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset; - iter.disk_name = bio->bi_disk->disk_name; + iter.disk_name = bio->bi_bdev->bd_disk->disk_name; iter.interval = 1 << bi->interval_exp; iter.seed = proc_iter->bi_sector; iter.prot_buf = prot_buf; @@ -208,8 +208,8 @@ static blk_status_t bio_integrity_process(struct bio *bio, bool bio_integrity_prep(struct bio *bio) { struct bio_integrity_payload *bip; - struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); - struct request_queue *q = bio->bi_disk->queue; + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); + struct request_queue *q = bio->bi_bdev->bd_disk->queue; void *buf; unsigned long start, end; unsigned int len, nr_pages; @@ -329,7 +329,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) struct bio_integrity_payload *bip = container_of(work, struct bio_integrity_payload, bip_work); struct bio *bio = bip->bip_bio; - struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); /* * At the moment verify is called bio's iterator was advanced @@ -355,7 +355,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) */ bool __bio_integrity_endio(struct bio *bio) { - struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); struct bio_integrity_payload *bip = bio_integrity(bio); if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && @@ -381,7 +381,7 @@ bool __bio_integrity_endio(struct bio *bio) void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) { struct bio_integrity_payload *bip = bio_integrity(bio); - struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9); bip->bip_iter.bi_sector += bytes_done >> 9; @@ -397,7 +397,7 @@ void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) void bio_integrity_trim(struct bio *bio) { struct bio_integrity_payload *bip = bio_integrity(bio); - struct blk_integrity *bi = blk_get_integrity(bio->bi_disk); + struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio)); } diff --git a/block/bio.c b/block/bio.c index 1f2cc1fbe283..0b70ade17da6 100644 --- a/block/bio.c +++ b/block/bio.c @@ -607,16 +607,7 @@ void bio_truncate(struct bio *bio, unsigned new_size) */ void guard_bio_eod(struct bio *bio) { - sector_t maxsector; - struct block_device *part; - - rcu_read_lock(); - part = __disk_get_part(bio->bi_disk, bio->bi_partno); - if (part) - maxsector = bdev_nr_sectors(part); - else - maxsector = get_capacity(bio->bi_disk); - rcu_read_unlock(); + sector_t maxsector = bdev_nr_sectors(bio->bi_bdev); if (!maxsector) return; @@ -676,11 +667,10 @@ void __bio_clone_fast(struct bio *bio, struct bio *bio_src) BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio)); /* - * most users will be overriding ->bi_disk with a new target, + * most users will be overriding ->bi_bdev with a new target, * so we don't set nor calculate new physical/hw segment counts here */ - bio->bi_disk = bio_src->bi_disk; - bio->bi_partno = bio_src->bi_partno; + bio->bi_bdev = bio_src->bi_bdev; bio_set_flag(bio, BIO_CLONED); if (bio_flagged(bio_src, BIO_THROTTLED)) bio_set_flag(bio, BIO_THROTTLED); @@ -730,7 +720,7 @@ EXPORT_SYMBOL(bio_clone_fast); const char *bio_devname(struct bio *bio, char *buf) { - return disk_name(bio->bi_disk, bio->bi_partno, buf); + return bdevname(bio->bi_bdev, buf); } EXPORT_SYMBOL(bio_devname); @@ -1037,7 +1027,7 @@ static int __bio_iov_append_get_pages(struct bio *bio, struct iov_iter *iter) { unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; unsigned int max_append_sectors = queue_max_zone_append_sectors(q); struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; struct page **pages = (struct page **)bv; @@ -1145,7 +1135,8 @@ static void submit_bio_wait_endio(struct bio *bio) */ int submit_bio_wait(struct bio *bio) { - DECLARE_COMPLETION_ONSTACK_MAP(done, bio->bi_disk->lockdep_map); + DECLARE_COMPLETION_ONSTACK_MAP(done, + bio->bi_bdev->bd_disk->lockdep_map); unsigned long hang_check; bio->bi_private = &done; @@ -1422,8 +1413,8 @@ again: if (!bio_integrity_endio(bio)) return; - if (bio->bi_disk) - rq_qos_done_bio(bio->bi_disk->queue, bio); + if (bio->bi_bdev) + rq_qos_done_bio(bio->bi_bdev->bd_disk->queue, bio); /* * Need to have a real endio function for chained bios, otherwise @@ -1438,8 +1429,8 @@ again: goto again; } - if (bio->bi_disk && bio_flagged(bio, BIO_TRACE_COMPLETION)) { - trace_block_bio_complete(bio->bi_disk->queue, bio); + if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) { + trace_block_bio_complete(bio->bi_bdev->bd_disk->queue, bio); bio_clear_flag(bio, BIO_TRACE_COMPLETION); } diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 031114d454a6..3465d6ee708e 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -1800,7 +1800,8 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, struct blkcg_gq *blkg, *ret_blkg = NULL; rcu_read_lock(); - blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_disk->queue); + blkg = blkg_lookup_create(css_to_blkcg(css), + bio->bi_bdev->bd_disk->queue); while (blkg) { if (blkg_tryget(blkg)) { ret_blkg = blkg; @@ -1836,8 +1837,8 @@ void bio_associate_blkg_from_css(struct bio *bio, if (css && css->parent) { bio->bi_blkg = blkg_tryget_closest(bio, css); } else { - blkg_get(bio->bi_disk->queue->root_blkg); - bio->bi_blkg = bio->bi_disk->queue->root_blkg; + blkg_get(bio->bi_bdev->bd_disk->queue->root_blkg); + bio->bi_blkg = bio->bi_bdev->bd_disk->queue->root_blkg; } } EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css); diff --git a/block/blk-core.c b/block/blk-core.c index 08ff8ca32529..a3a54cd86c9c 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -476,7 +476,7 @@ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) static inline int bio_queue_enter(struct bio *bio) { - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; bool nowait = bio->bi_opf & REQ_NOWAIT; int ret; @@ -712,7 +712,7 @@ static inline bool bio_check_ro(struct bio *bio, struct block_device *part) static noinline int should_fail_bio(struct bio *bio) { - if (should_fail_request(bio->bi_disk->part0, bio->bi_iter.bi_size)) + if (should_fail_request(bdev_whole(bio->bi_bdev), bio->bi_iter.bi_size)) return -EIO; return 0; } @@ -741,13 +741,9 @@ static inline int bio_check_eod(struct bio *bio, sector_t maxsector) */ static inline int blk_partition_remap(struct bio *bio) { - struct block_device *p; + struct block_device *p = bio->bi_bdev; int ret = -EIO; - rcu_read_lock(); - p = __disk_get_part(bio->bi_disk, bio->bi_partno); - if (unlikely(!p)) - goto out; if (unlikely(should_fail_request(p, bio->bi_iter.bi_size))) goto out; if (unlikely(bio_check_ro(bio, p))) @@ -761,10 +757,9 @@ static inline int blk_partition_remap(struct bio *bio) bio->bi_iter.bi_sector - p->bd_start_sect); } - bio->bi_partno = 0; + bio->bi_bdev = bdev_whole(p); ret = 0; out: - rcu_read_unlock(); return ret; } @@ -805,7 +800,8 @@ static inline blk_status_t blk_check_zone_append(struct request_queue *q, static noinline_for_stack bool submit_bio_checks(struct bio *bio) { - struct request_queue *q = bio->bi_disk->queue; + struct block_device *bdev = bio->bi_bdev; + struct request_queue *q = bdev->bd_disk->queue; blk_status_t status = BLK_STS_IOERR; struct blk_plug *plug; @@ -825,13 +821,13 @@ static noinline_for_stack bool submit_bio_checks(struct bio *bio) if (should_fail_bio(bio)) goto end_io; - if (bio->bi_partno) { + if (bio->bi_bdev->bd_partno) { if (unlikely(blk_partition_remap(bio))) goto end_io; } else { - if (unlikely(bio_check_ro(bio, bio->bi_disk->part0))) + if (unlikely(bio_check_ro(bio, bdev_whole(bdev)))) goto end_io; - if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk)))) + if (unlikely(bio_check_eod(bio, get_capacity(bdev->bd_disk)))) goto end_io; } @@ -924,7 +920,7 @@ end_io: static blk_qc_t __submit_bio(struct bio *bio) { - struct gendisk *disk = bio->bi_disk; + struct gendisk *disk = bio->bi_bdev->bd_disk; blk_qc_t ret = BLK_QC_T_NONE; if (blk_crypto_bio_prep(&bio)) { @@ -966,7 +962,7 @@ static blk_qc_t __submit_bio_noacct(struct bio *bio) current->bio_list = bio_list_on_stack; do { - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; struct bio_list lower, same; if (unlikely(bio_queue_enter(bio) != 0)) @@ -987,7 +983,7 @@ static blk_qc_t __submit_bio_noacct(struct bio *bio) bio_list_init(&lower); bio_list_init(&same); while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL) - if (q == bio->bi_disk->queue) + if (q == bio->bi_bdev->bd_disk->queue) bio_list_add(&same, bio); else bio_list_add(&lower, bio); @@ -1012,7 +1008,7 @@ static blk_qc_t __submit_bio_noacct_mq(struct bio *bio) current->bio_list = bio_list; do { - struct gendisk *disk = bio->bi_disk; + struct gendisk *disk = bio->bi_bdev->bd_disk; if (unlikely(bio_queue_enter(bio) != 0)) continue; @@ -1055,7 +1051,7 @@ blk_qc_t submit_bio_noacct(struct bio *bio) return BLK_QC_T_NONE; } - if (!bio->bi_disk->fops->submit_bio) + if (!bio->bi_bdev->bd_disk->fops->submit_bio) return __submit_bio_noacct_mq(bio); return __submit_bio_noacct(bio); } @@ -1067,7 +1063,7 @@ EXPORT_SYMBOL(submit_bio_noacct); * * submit_bio() is used to submit I/O requests to block devices. It is passed a * fully set up &struct bio that describes the I/O that needs to be done. The - * bio will be send to the device described by the bi_disk and bi_partno fields. + * bio will be send to the device described by the bi_bdev field. * * The success/failure status of the request, along with notification of * completion, is delivered asynchronously through the ->bi_end_io() callback @@ -1087,7 +1083,8 @@ blk_qc_t submit_bio(struct bio *bio) unsigned int count; if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME)) - count = queue_logical_block_size(bio->bi_disk->queue) >> 9; + count = queue_logical_block_size( + bio->bi_bdev->bd_disk->queue) >> 9; else count = bio_sectors(bio); diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index c162b754efbd..8f1e18176731 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -167,7 +167,7 @@ static struct bio *blk_crypto_clone_bio(struct bio *bio_src) bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL); if (!bio) return NULL; - bio->bi_disk = bio_src->bi_disk; + bio->bi_bdev = bio_src->bi_bdev; bio->bi_opf = bio_src->bi_opf; bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 5da43f0973b4..09fcb18fa778 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -280,7 +280,7 @@ bool __blk_crypto_bio_prep(struct bio **bio_ptr) * Success if device supports the encryption context, or if we succeeded * in falling back to the crypto API. */ - if (blk_ksm_crypto_cfg_supported(bio->bi_disk->queue->ksm, + if (blk_ksm_crypto_cfg_supported(bio->bi_bdev->bd_disk->queue->ksm, &bc_key->crypto_cfg)) return true; diff --git a/block/blk-merge.c b/block/blk-merge.c index 808768f6b174..ffb4aa0ea68b 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -298,14 +298,13 @@ split: * Split a bio into two bios, chain the two bios, submit the second half and * store a pointer to the first half in *@bio. If the second bio is still too * big it will be split by a recursive call to this function. Since this - * function may allocate a new bio from @bio->bi_disk->queue->bio_split, it is - * the responsibility of the caller to ensure that - * @bio->bi_disk->queue->bio_split is only released after processing of the - * split bio has finished. + * function may allocate a new bio from q->bio_split, it is the responsibility + * of the caller to ensure that q->bio_split is only released after processing + * of the split bio has finished. */ void __blk_queue_split(struct bio **bio, unsigned int *nr_segs) { - struct request_queue *q = (*bio)->bi_disk->queue; + struct request_queue *q = (*bio)->bi_bdev->bd_disk->queue; struct bio *split = NULL; switch (bio_op(*bio)) { @@ -358,9 +357,9 @@ void __blk_queue_split(struct bio **bio, unsigned int *nr_segs) * * Split a bio into two bios, chains the two bios, submit the second half and * store a pointer to the first half in *@bio. Since this function may allocate - * a new bio from @bio->bi_disk->queue->bio_split, it is the responsibility of - * the caller to ensure that @bio->bi_disk->queue->bio_split is only released - * after processing of the split bio has finished. + * a new bio from q->bio_split, it is the responsibility of the caller to ensure + * that q->bio_split is only released after processing of the split bio has + * finished. */ void blk_queue_split(struct bio **bio) { @@ -866,7 +865,7 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) return false; /* must be same device */ - if (rq->rq_disk != bio->bi_disk) + if (rq->rq_disk != bio->bi_bdev->bd_disk) return false; /* only merge integrity protected bio into ditto rq */ diff --git a/block/blk-mq.c b/block/blk-mq.c index f285a9123a8b..74b17b396f4c 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2128,7 +2128,7 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) */ blk_qc_t blk_mq_submit_bio(struct bio *bio) { - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; const int is_sync = op_is_sync(bio->bi_opf); const int is_flush_fua = op_is_flush(bio->bi_opf); struct blk_mq_alloc_data data = { diff --git a/block/blk-throttle.c b/block/blk-throttle.c index d52cac9f3a7c..b1b22d863bdf 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -2178,7 +2178,7 @@ static inline void throtl_update_latency_buckets(struct throtl_data *td) bool blk_throtl_bio(struct bio *bio) { - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; struct blkcg_gq *blkg = bio->bi_blkg; struct throtl_qnode *qn = NULL; struct throtl_grp *tg = blkg_to_tg(blkg); diff --git a/block/blk.h b/block/blk.h index 7550364c326c..10ab7c0d0766 100644 --- a/block/blk.h +++ b/block/blk.h @@ -202,8 +202,6 @@ static inline void elevator_exit(struct request_queue *q, __elevator_exit(q, e); } -struct block_device *__disk_get_part(struct gendisk *disk, int partno); - ssize_t part_size_show(struct device *dev, struct device_attribute *attr, char *buf); ssize_t part_stat_show(struct device *dev, struct device_attribute *attr, diff --git a/block/bounce.c b/block/bounce.c index d3f51acd6e3b..a22a8a1942b2 100644 --- a/block/bounce.c +++ b/block/bounce.c @@ -246,7 +246,7 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs); if (!bio) return NULL; - bio->bi_disk = bio_src->bi_disk; + bio->bi_bdev = bio_src->bi_bdev; bio->bi_opf = bio_src->bi_opf; bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; diff --git a/block/genhd.c b/block/genhd.c index ca5d880af512..e536d0b4bbae 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -161,7 +161,7 @@ static void part_in_flight_rw(struct block_device *part, inflight[1] = 0; } -struct block_device *__disk_get_part(struct gendisk *disk, int partno) +static struct block_device *__disk_get_part(struct gendisk *disk, int partno) { struct disk_part_tbl *ptbl = rcu_dereference(disk->part_tbl); diff --git a/drivers/block/brd.c b/drivers/block/brd.c index c7c821419079..18bf99906662 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -284,7 +284,7 @@ out: static blk_qc_t brd_submit_bio(struct bio *bio) { - struct brd_device *brd = bio->bi_disk->private_data; + struct brd_device *brd = bio->bi_bdev->bd_disk->private_data; sector_t sector = bio->bi_iter.bi_sector; struct bio_vec bvec; struct bvec_iter iter; diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index 8f879e5c2f67..b2c93a29c251 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h @@ -1579,8 +1579,8 @@ static inline void drbd_submit_bio_noacct(struct drbd_device *device, int fault_type, struct bio *bio) { __release(local); - if (!bio->bi_disk) { - drbd_err(device, "drbd_submit_bio_noacct: bio->bi_disk == NULL\n"); + if (!bio->bi_bdev) { + drbd_err(device, "drbd_submit_bio_noacct: bio->bi_bdev == NULL\n"); bio->bi_status = BLK_STS_IOERR; bio_endio(bio); return; diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index 330f851cb8f0..ea0f31ab3343 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c @@ -1595,7 +1595,7 @@ void do_submit(struct work_struct *ws) blk_qc_t drbd_submit_bio(struct bio *bio) { - struct drbd_device *device = bio->bi_disk->private_data; + struct drbd_device *device = bio->bi_bdev->bd_disk->private_data; unsigned long start_jif; blk_queue_split(&bio); diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 5357c3a4a36f..d6c821d48090 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -1420,7 +1420,7 @@ static blk_qc_t null_submit_bio(struct bio *bio) { sector_t sector = bio->bi_iter.bi_sector; sector_t nr_sectors = bio_sectors(bio); - struct nullb *nullb = bio->bi_disk->private_data; + struct nullb *nullb = bio->bi_bdev->bd_disk->private_data; struct nullb_queue *nq = nullb_to_queue(nullb); struct nullb_cmd *cmd; diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index b8bb8ec7538d..658a0981cb54 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -2374,7 +2374,7 @@ static blk_qc_t pkt_submit_bio(struct bio *bio) blk_queue_split(&bio); - pd = bio->bi_disk->queue->queuedata; + pd = bio->bi_bdev->bd_disk->queue->queuedata; if (!pd) { pr_err("%s incorrect request queue\n", bio_devname(bio, b)); goto end_io; @@ -2418,7 +2418,7 @@ static blk_qc_t pkt_submit_bio(struct bio *bio) split = bio; } - pkt_make_request_write(bio->bi_disk->queue, split); + pkt_make_request_write(bio->bi_bdev->bd_disk->queue, split); } while (split != bio); return BLK_QC_T_NONE; diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c index b71d28372ef3..1d738999fb69 100644 --- a/drivers/block/ps3vram.c +++ b/drivers/block/ps3vram.c @@ -581,7 +581,7 @@ out: static blk_qc_t ps3vram_submit_bio(struct bio *bio) { - struct ps3_system_bus_device *dev = bio->bi_disk->private_data; + struct ps3_system_bus_device *dev = bio->bi_bdev->bd_disk->private_data; struct ps3vram_priv *priv = ps3_system_bus_get_drvdata(dev); int busy; diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c index edacefff6e35..9a28322a8cd8 100644 --- a/drivers/block/rsxx/dev.c +++ b/drivers/block/rsxx/dev.c @@ -122,7 +122,7 @@ static void bio_dma_done_cb(struct rsxx_cardinfo *card, static blk_qc_t rsxx_submit_bio(struct bio *bio) { - struct rsxx_cardinfo *card = bio->bi_disk->private_data; + struct rsxx_cardinfo *card = bio->bi_bdev->bd_disk->private_data; struct rsxx_bio_meta *bio_meta; blk_status_t st = BLK_STS_IOERR; diff --git a/drivers/block/umem.c b/drivers/block/umem.c index 2b95d7b33b91..982732dbe82e 100644 --- a/drivers/block/umem.c +++ b/drivers/block/umem.c @@ -521,7 +521,7 @@ static int mm_check_plugged(struct cardinfo *card) static blk_qc_t mm_submit_bio(struct bio *bio) { - struct cardinfo *card = bio->bi_disk->private_data; + struct cardinfo *card = bio->bi_bdev->bd_disk->private_data; pr_debug("mm_make_request %llu %u\n", (unsigned long long)bio->bi_iter.bi_sector, diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index e2933cb7a82a..d6243dbc53cc 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1596,7 +1596,7 @@ static void __zram_make_request(struct zram *zram, struct bio *bio) */ static blk_qc_t zram_submit_bio(struct bio *bio) { - struct zram *zram = bio->bi_disk->private_data; + struct zram *zram = bio->bi_bdev->bd_disk->private_data; if (!valid_io_request(zram, bio->bi_iter.bi_sector, bio->bi_iter.bi_size)) { diff --git a/drivers/lightnvm/pblk-init.c b/drivers/lightnvm/pblk-init.c index b6246f73895c..5924f09c217b 100644 --- a/drivers/lightnvm/pblk-init.c +++ b/drivers/lightnvm/pblk-init.c @@ -49,7 +49,7 @@ struct bio_set pblk_bio_set; static blk_qc_t pblk_submit_bio(struct bio *bio) { - struct pblk *pblk = bio->bi_disk->queue->queuedata; + struct pblk *pblk = bio->bi_bdev->bd_disk->queue->queuedata; if (bio_op(bio) == REQ_OP_DISCARD) { pblk_discard(pblk, bio); diff --git a/drivers/md/bcache/debug.c b/drivers/md/bcache/debug.c index b00fd08d696b..058dd8014428 100644 --- a/drivers/md/bcache/debug.c +++ b/drivers/md/bcache/debug.c @@ -114,7 +114,7 @@ void bch_data_verify(struct cached_dev *dc, struct bio *bio) check = bio_kmalloc(GFP_NOIO, bio_segments(bio)); if (!check) return; - check->bi_disk = bio->bi_disk; + check->bi_bdev = bio->bi_bdev; check->bi_opf = REQ_OP_READ; check->bi_iter.bi_sector = bio->bi_iter.bi_sector; check->bi_iter.bi_size = bio->bi_iter.bi_size; diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index 85b1f2a9b72d..dfc35d6d05ed 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -894,7 +894,8 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s, !(bio->bi_opf & (REQ_META|REQ_PRIO)) && s->iop.c->gc_stats.in_use < CUTOFF_CACHE_READA) reada = min_t(sector_t, dc->readahead >> 9, - get_capacity(bio->bi_disk) - bio_end_sector(bio)); + get_capacity(bio->bi_bdev->bd_disk) - + bio_end_sector(bio)); s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada); @@ -1167,7 +1168,7 @@ static void quit_max_writeback_rate(struct cache_set *c, blk_qc_t cached_dev_submit_bio(struct bio *bio) { struct search *s; - struct bcache_device *d = bio->bi_disk->private_data; + struct bcache_device *d = bio->bi_bdev->bd_disk->private_data; struct cached_dev *dc = container_of(d, struct cached_dev, disk); int rw = bio_data_dir(bio); @@ -1274,7 +1275,7 @@ blk_qc_t flash_dev_submit_bio(struct bio *bio) { struct search *s; struct closure *cl; - struct bcache_device *d = bio->bi_disk->private_data; + struct bcache_device *d = bio->bi_bdev->bd_disk->private_data; if (unlikely(d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags))) { bio->bi_status = BLK_STS_IOERR; diff --git a/drivers/md/dm-bio-record.h b/drivers/md/dm-bio-record.h index 2ea0360108e1..a3b71350eec8 100644 --- a/drivers/md/dm-bio-record.h +++ b/drivers/md/dm-bio-record.h @@ -18,8 +18,7 @@ */ struct dm_bio_details { - struct gendisk *bi_disk; - u8 bi_partno; + struct block_device *bi_bdev; int __bi_remaining; unsigned long bi_flags; struct bvec_iter bi_iter; @@ -31,8 +30,7 @@ struct dm_bio_details { static inline void dm_bio_record(struct dm_bio_details *bd, struct bio *bio) { - bd->bi_disk = bio->bi_disk; - bd->bi_partno = bio->bi_partno; + bd->bi_bdev = bio->bi_bdev; bd->bi_flags = bio->bi_flags; bd->bi_iter = bio->bi_iter; bd->__bi_remaining = atomic_read(&bio->__bi_remaining); @@ -44,8 +42,7 @@ static inline void dm_bio_record(struct dm_bio_details *bd, struct bio *bio) static inline void dm_bio_restore(struct dm_bio_details *bd, struct bio *bio) { - bio->bi_disk = bd->bi_disk; - bio->bi_partno = bd->bi_partno; + bio->bi_bdev = bd->bi_bdev; bio->bi_flags = bd->bi_flags; bio->bi_iter = bd->bi_iter; atomic_set(&bio->__bi_remaining, bd->__bi_remaining); diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index fa09bc4e4c54..b0a82f29a2e4 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -145,7 +145,7 @@ static void dispatch_bios(void *context, struct bio_list *bio_list) struct dm_raid1_bio_record { struct mirror *m; - /* if details->bi_disk == NULL, details were not saved */ + /* if details->bi_bdev == NULL, details were not saved */ struct dm_bio_details details; region_t write_region; }; @@ -1190,7 +1190,7 @@ static int mirror_map(struct dm_target *ti, struct bio *bio) struct dm_raid1_bio_record *bio_record = dm_per_bio_data(bio, sizeof(struct dm_raid1_bio_record)); - bio_record->details.bi_disk = NULL; + bio_record->details.bi_bdev = NULL; if (rw == WRITE) { /* Save region for mirror_end_io() handler */ @@ -1257,7 +1257,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, goto out; if (unlikely(*error)) { - if (!bio_record->details.bi_disk) { + if (!bio_record->details.bi_bdev) { /* * There wasn't enough memory to record necessary * information for a retry or there was no other @@ -1282,7 +1282,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, bd = &bio_record->details; dm_bio_restore(bd, bio); - bio_record->details.bi_disk = NULL; + bio_record->details.bi_bdev = NULL; bio->bi_status = 0; queue_bio(ms, bio, rw); @@ -1292,7 +1292,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, } out: - bio_record->details.bi_disk = NULL; + bio_record->details.bi_bdev = NULL; return DM_ENDIO_DONE; } diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 7bac564f3faa..479ec5bea09e 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -977,16 +977,17 @@ static void clone_endio(struct bio *bio) struct mapped_device *md = tio->io->md; dm_endio_fn endio = tio->ti->type->end_io; struct bio *orig_bio = io->orig_bio; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; if (unlikely(error == BLK_STS_TARGET)) { if (bio_op(bio) == REQ_OP_DISCARD && - !bio->bi_disk->queue->limits.max_discard_sectors) + !q->limits.max_discard_sectors) disable_discard(md); else if (bio_op(bio) == REQ_OP_WRITE_SAME && - !bio->bi_disk->queue->limits.max_write_same_sectors) + !q->limits.max_write_same_sectors) disable_write_same(md); else if (bio_op(bio) == REQ_OP_WRITE_ZEROES && - !bio->bi_disk->queue->limits.max_write_zeroes_sectors) + !q->limits.max_write_zeroes_sectors) disable_write_zeroes(md); } @@ -996,7 +997,7 @@ static void clone_endio(struct bio *bio) */ if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) { sector_t written_sector = bio->bi_iter.bi_sector; - struct request_queue *q = orig_bio->bi_disk->queue; + struct request_queue *q = orig_bio->bi_bdev->bd_disk->queue; u64 mask = (u64)blk_queue_zone_sectors(q) - 1; orig_bio->bi_iter.bi_sector += written_sector & mask; @@ -1422,8 +1423,7 @@ static int __send_empty_flush(struct clone_info *ci) */ bio_init(&flush_bio, NULL, 0); flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC; - flush_bio.bi_disk = ci->io->md->disk; - bio_associate_blkg(&flush_bio); + bio_set_dev(&flush_bio, ci->io->md->disk->part0); ci->bio = &flush_bio; ci->sector_count = 0; @@ -1626,7 +1626,7 @@ static blk_qc_t __split_and_process_bio(struct mapped_device *md, static blk_qc_t dm_submit_bio(struct bio *bio) { - struct mapped_device *md = bio->bi_disk->private_data; + struct mapped_device *md = bio->bi_bdev->bd_disk->private_data; blk_qc_t ret = BLK_QC_T_NONE; int srcu_idx; struct dm_table *map; diff --git a/drivers/md/md-linear.c b/drivers/md/md-linear.c index 68cac7d19278..63ed8329a98d 100644 --- a/drivers/md/md-linear.c +++ b/drivers/md/md-linear.c @@ -252,7 +252,7 @@ static bool linear_make_request(struct mddev *mddev, struct bio *bio) start_sector + data_offset; if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && - !blk_queue_discard(bio->bi_disk->queue))) { + !blk_queue_discard(bio->bi_bdev->bd_disk->queue))) { /* Just ignore it */ bio_endio(bio); } else { diff --git a/drivers/md/md.c b/drivers/md/md.c index 04384452a7ab..cf06dbb1aa53 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -486,7 +486,7 @@ static void md_end_io(struct bio *bio) static blk_qc_t md_submit_bio(struct bio *bio) { const int rw = bio_data_dir(bio); - struct mddev *mddev = bio->bi_disk->private_data; + struct mddev *mddev = bio->bi_bdev->bd_disk->private_data; if (mddev == NULL || mddev->pers == NULL) { bio_io_error(bio); diff --git a/drivers/md/md.h b/drivers/md/md.h index 34070ab30a8a..f13290ccc1c2 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -556,7 +556,7 @@ static inline void md_sync_acct(struct block_device *bdev, unsigned long nr_sect static inline void md_sync_acct_bio(struct bio *bio, unsigned long nr_sectors) { - atomic_add(nr_sectors, &bio->bi_disk->sync_io); + md_sync_acct(bio->bi_bdev, nr_sectors); } struct md_personality @@ -793,14 +793,14 @@ static inline void mddev_clear_unsupported_flags(struct mddev *mddev, static inline void mddev_check_writesame(struct mddev *mddev, struct bio *bio) { if (bio_op(bio) == REQ_OP_WRITE_SAME && - !bio->bi_disk->queue->limits.max_write_same_sectors) + !bio->bi_bdev->bd_disk->queue->limits.max_write_same_sectors) mddev->queue->limits.max_write_same_sectors = 0; } static inline void mddev_check_write_zeroes(struct mddev *mddev, struct bio *bio) { if (bio_op(bio) == REQ_OP_WRITE_ZEROES && - !bio->bi_disk->queue->limits.max_write_zeroes_sectors) + !bio->bi_bdev->bd_disk->queue->limits.max_write_zeroes_sectors) mddev->queue->limits.max_write_zeroes_sectors = 0; } diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index c0347997f6ff..3b19141cdb4b 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -794,13 +794,13 @@ static void flush_bio_list(struct r1conf *conf, struct bio *bio) while (bio) { /* submit pending writes */ struct bio *next = bio->bi_next; - struct md_rdev *rdev = (void *)bio->bi_disk; + struct md_rdev *rdev = (void *)bio->bi_bdev; bio->bi_next = NULL; bio_set_dev(bio, rdev->bdev); if (test_bit(Faulty, &rdev->flags)) { bio_io_error(bio); } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && - !blk_queue_discard(bio->bi_disk->queue))) + !blk_queue_discard(bio->bi_bdev->bd_disk->queue))) /* Just ignore it */ bio_endio(bio); else @@ -1520,7 +1520,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, trace_block_bio_remap(mbio, disk_devt(mddev->gendisk), r1_bio->sector); /* flush_pending_writes() needs access to the rdev so...*/ - mbio->bi_disk = (void *)conf->mirrors[i].rdev; + mbio->bi_bdev = (void *)conf->mirrors[i].rdev; cb = blk_check_plugged(raid1_unplug, mddev, sizeof(*plug)); if (cb) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index c5d88ef6a45c..be8f14afb6d1 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -882,13 +882,13 @@ static void flush_pending_writes(struct r10conf *conf) while (bio) { /* submit pending writes */ struct bio *next = bio->bi_next; - struct md_rdev *rdev = (void*)bio->bi_disk; + struct md_rdev *rdev = (void*)bio->bi_bdev; bio->bi_next = NULL; bio_set_dev(bio, rdev->bdev); if (test_bit(Faulty, &rdev->flags)) { bio_io_error(bio); } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && - !blk_queue_discard(bio->bi_disk->queue))) + !blk_queue_discard(bio->bi_bdev->bd_disk->queue))) /* Just ignore it */ bio_endio(bio); else @@ -1075,13 +1075,13 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) while (bio) { /* submit pending writes */ struct bio *next = bio->bi_next; - struct md_rdev *rdev = (void*)bio->bi_disk; + struct md_rdev *rdev = (void*)bio->bi_bdev; bio->bi_next = NULL; bio_set_dev(bio, rdev->bdev); if (test_bit(Faulty, &rdev->flags)) { bio_io_error(bio); } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && - !blk_queue_discard(bio->bi_disk->queue))) + !blk_queue_discard(bio->bi_bdev->bd_disk->queue))) /* Just ignore it */ bio_endio(bio); else @@ -1253,7 +1253,7 @@ static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio, trace_block_bio_remap(mbio, disk_devt(conf->mddev->gendisk), r10_bio->sector); /* flush_pending_writes() needs access to the rdev so...*/ - mbio->bi_disk = (void *)rdev; + mbio->bi_bdev = (void *)rdev; atomic_inc(&r10_bio->remaining); @@ -3003,7 +3003,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, /* Again, very different code for resync and recovery. * Both must result in an r10bio with a list of bios that - * have bi_end_io, bi_sector, bi_disk set, + * have bi_end_io, bi_sector, bi_bdev set, * and bi_private set to the r10bio. * For recovery, we may actually create several r10bios * with 2 bios in each, that correspond to the bios in the main one. diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 3a90cc0e43ca..f411b9e5c332 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -5310,7 +5310,7 @@ static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) unsigned int chunk_sectors; unsigned int bio_sectors = bio_sectors(bio); - WARN_ON_ONCE(bio->bi_partno); + WARN_ON_ONCE(bio->bi_bdev->bd_partno); chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors); return chunk_sectors >= diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c index 22e5617b2cea..e03a1f38d750 100644 --- a/drivers/nvdimm/blk.c +++ b/drivers/nvdimm/blk.c @@ -165,7 +165,7 @@ static int nsblk_do_bvec(struct nd_namespace_blk *nsblk, static blk_qc_t nd_blk_submit_bio(struct bio *bio) { struct bio_integrity_payload *bip; - struct nd_namespace_blk *nsblk = bio->bi_disk->private_data; + struct nd_namespace_blk *nsblk = bio->bi_bdev->bd_disk->private_data; struct bvec_iter iter; unsigned long start; struct bio_vec bvec; @@ -177,7 +177,7 @@ static blk_qc_t nd_blk_submit_bio(struct bio *bio) bip = bio_integrity(bio); rw = bio_data_dir(bio); - do_acct = blk_queue_io_stat(bio->bi_disk->queue); + do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue); if (do_acct) start = bio_start_io_acct(bio); bio_for_each_segment(bvec, bio, iter) { diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 12ff6f8784ac..41aa1f01fc07 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -1442,7 +1442,7 @@ static int btt_do_bvec(struct btt *btt, struct bio_integrity_payload *bip, static blk_qc_t btt_submit_bio(struct bio *bio) { struct bio_integrity_payload *bip = bio_integrity(bio); - struct btt *btt = bio->bi_disk->private_data; + struct btt *btt = bio->bi_bdev->bd_disk->private_data; struct bvec_iter iter; unsigned long start; struct bio_vec bvec; @@ -1452,7 +1452,7 @@ static blk_qc_t btt_submit_bio(struct bio *bio) if (!bio_integrity_prep(bio)) return BLK_QC_T_NONE; - do_acct = blk_queue_io_stat(bio->bi_disk->queue); + do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue); if (do_acct) start = bio_start_io_acct(bio); bio_for_each_segment(bvec, bio, iter) { diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 875076b0ea6c..72740835c85c 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -197,13 +197,13 @@ static blk_qc_t pmem_submit_bio(struct bio *bio) unsigned long start; struct bio_vec bvec; struct bvec_iter iter; - struct pmem_device *pmem = bio->bi_disk->private_data; + struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data; struct nd_region *nd_region = to_region(pmem); if (bio->bi_opf & REQ_PREFLUSH) ret = nvdimm_flush(nd_region, bio); - do_acct = blk_queue_io_stat(bio->bi_disk->queue); + do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue); if (do_acct) start = bio_start_io_acct(bio); bio_for_each_segment(bvec, bio, iter) { diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 566788ba4e7d..a39befb4deba 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1113,7 +1113,7 @@ static int nvme_submit_user_cmd(struct request_queue *q, { bool write = nvme_is_write(cmd); struct nvme_ns *ns = q->queuedata; - struct gendisk *disk = ns ? ns->disk : NULL; + struct block_device *bdev = ns ? ns->disk->part0 : NULL; struct request *req; struct bio *bio = NULL; void *meta = NULL; @@ -1133,8 +1133,8 @@ static int nvme_submit_user_cmd(struct request_queue *q, if (ret) goto out; bio = req->bio; - bio->bi_disk = disk; - if (disk && meta_buffer && meta_len) { + bio->bi_bdev = bdev; + if (bdev && meta_buffer && meta_len) { meta = nvme_add_user_metadata(bio, meta_buffer, meta_len, meta_seed, write); if (IS_ERR(meta)) { diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c index 470cef3abec3..6c8eab8de288 100644 --- a/drivers/nvme/host/lightnvm.c +++ b/drivers/nvme/host/lightnvm.c @@ -757,7 +757,6 @@ static int nvme_nvm_submit_user_cmd(struct request_queue *q, { bool write = nvme_is_write((struct nvme_command *)vcmd); struct nvm_dev *dev = ns->ndev; - struct gendisk *disk = ns->disk; struct request *rq; struct bio *bio = NULL; __le64 *ppa_list = NULL; @@ -817,7 +816,7 @@ static int nvme_nvm_submit_user_cmd(struct request_queue *q, vcmd->ph_rw.metadata = cpu_to_le64(metadata_dma); } - bio->bi_disk = disk; + bio->bi_bdev = ns->disk->part0; } blk_execute_rq(q, NULL, rq, 0); diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 9ac762b28811..a6d44e7a775f 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -296,7 +296,7 @@ static bool nvme_available_path(struct nvme_ns_head *head) blk_qc_t nvme_ns_head_submit_bio(struct bio *bio) { - struct nvme_ns_head *head = bio->bi_disk->private_data; + struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data; struct device *dev = disk_to_dev(head->disk); struct nvme_ns *ns; blk_qc_t ret = BLK_QC_T_NONE; @@ -312,7 +312,7 @@ blk_qc_t nvme_ns_head_submit_bio(struct bio *bio) srcu_idx = srcu_read_lock(&head->srcu); ns = nvme_find_path(head); if (likely(ns)) { - bio->bi_disk = ns->disk; + bio->bi_bdev = ns->disk->part0; bio->bi_opf |= REQ_NVME_MPATH; trace_block_bio_remap(bio, disk_devt(ns->head->disk), bio->bi_iter.bi_sector); @@ -352,7 +352,7 @@ static void nvme_requeue_work(struct work_struct *work) * Reset disk to the mpath node and resubmit to select a new * path. */ - bio->bi_disk = head->disk; + bio->bi_bdev = head->disk->part0; submit_bio_noacct(bio); } } diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index b7ce4f221d99..f5ef3edeb2fd 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -1468,7 +1468,7 @@ static int nvme_rdma_map_sg_pi(struct nvme_rdma_queue *queue, if (unlikely(nr)) goto mr_put; - nvme_rdma_set_sig_attrs(blk_get_integrity(bio->bi_disk), c, + nvme_rdma_set_sig_attrs(blk_get_integrity(bio->bi_bdev->bd_disk), c, req->mr->sig_attrs, ns->pi_type); nvme_rdma_set_prot_checks(c, &req->mr->sig_attrs->check_mask); diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index 5c5cff3f2374..da33cb4cba28 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -879,7 +879,7 @@ dcssblk_submit_bio(struct bio *bio) blk_queue_split(&bio); bytes_done = 0; - dev_info = bio->bi_disk->private_data; + dev_info = bio->bi_bdev->bd_disk->private_data; if (dev_info == NULL) goto fail; if ((bio->bi_iter.bi_sector & 7) != 0 || diff --git a/drivers/s390/block/xpram.c b/drivers/s390/block/xpram.c index c2536f7767b3..d1ed39162943 100644 --- a/drivers/s390/block/xpram.c +++ b/drivers/s390/block/xpram.c @@ -184,7 +184,7 @@ static unsigned long xpram_highest_page_index(void) */ static blk_qc_t xpram_submit_bio(struct bio *bio) { - xpram_device_t *xdev = bio->bi_disk->private_data; + xpram_device_t *xdev = bio->bi_bdev->bd_disk->private_data; struct bio_vec bvec; struct bvec_iter iter; unsigned int index; diff --git a/fs/btrfs/check-integrity.c b/fs/btrfs/check-integrity.c index 6ff44e53814c..113cb85c1fd4 100644 --- a/fs/btrfs/check-integrity.c +++ b/fs/btrfs/check-integrity.c @@ -2674,7 +2674,7 @@ static void __btrfsic_submit_bio(struct bio *bio) mutex_lock(&btrfsic_mutex); /* since btrfsic_submit_bio() is also called before * btrfsic_mount(), this might return NULL */ - dev_state = btrfsic_dev_state_lookup(bio_dev(bio) + bio->bi_partno); + dev_state = btrfsic_dev_state_lookup(bio->bi_bdev->bd_dev); if (NULL != dev_state && (bio_op(bio) == REQ_OP_WRITE) && bio_has_data(bio)) { unsigned int i = 0; @@ -2690,9 +2690,9 @@ static void __btrfsic_submit_bio(struct bio *bio) bio_is_patched = 0; if (dev_state->state->print_mask & BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH) - pr_info("submit_bio(rw=%d,0x%x, bi_vcnt=%u, bi_sector=%llu (bytenr %llu), bi_disk=%p)\n", + pr_info("submit_bio(rw=%d,0x%x, bi_vcnt=%u, bi_sector=%llu (bytenr %llu), bi_bdev=%p)\n", bio_op(bio), bio->bi_opf, segs, - bio->bi_iter.bi_sector, dev_bytenr, bio->bi_disk); + bio->bi_iter.bi_sector, dev_bytenr, bio->bi_bdev); mapped_datav = kmalloc_array(segs, sizeof(*mapped_datav), GFP_NOFS); @@ -2721,8 +2721,8 @@ static void __btrfsic_submit_bio(struct bio *bio) } else if (NULL != dev_state && (bio->bi_opf & REQ_PREFLUSH)) { if (dev_state->state->print_mask & BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH) - pr_info("submit_bio(rw=%d,0x%x FLUSH, disk=%p)\n", - bio_op(bio), bio->bi_opf, bio->bi_disk); + pr_info("submit_bio(rw=%d,0x%x FLUSH, bdev=%p)\n", + bio_op(bio), bio->bi_opf, bio->bi_bdev); if (!dev_state->dummy_block_for_bio_bh_flush.is_iodone) { if ((dev_state->state->print_mask & (BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH | diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 93fbf87bdc8d..b2204a2942cb 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -1105,8 +1105,7 @@ static int rbio_add_io_page(struct btrfs_raid_bio *rbio, * devices or if they are not contiguous */ if (last_end == disk_start && !last->bi_status && - last->bi_disk == stripe->dev->bdev->bd_disk && - last->bi_partno == stripe->dev->bdev->bd_partno) { + last->bi_bdev == stripe->dev->bdev) { ret = bio_add_page(last, page, PAGE_SIZE, 0); if (ret == PAGE_SIZE) return 0; @@ -1357,9 +1356,7 @@ static int find_bio_stripe(struct btrfs_raid_bio *rbio, for (i = 0; i < rbio->bbio->num_stripes; i++) { stripe = &rbio->bbio->stripes[i]; if (in_range(physical, stripe->physical, rbio->stripe_len) && - stripe->dev->bdev && - bio->bi_disk == stripe->dev->bdev->bd_disk && - bio->bi_partno == stripe->dev->bdev->bd_partno) { + stripe->dev->bdev && bio->bi_bdev == stripe->dev->bdev) { return i; } } diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 5f4f88a4d2c8..33f8f0f108bf 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -1695,7 +1695,7 @@ static void scrub_wr_submit(struct scrub_ctx *sctx) sbio = sctx->wr_curr_bio; sctx->wr_curr_bio = NULL; - WARN_ON(!sbio->bio->bi_disk); + WARN_ON(!sbio->bio->bi_bdev); scrub_pending_bio_inc(sctx); /* process all writes in a single worker thread. Then the block layer * orders the requests before sending them to the driver which diff --git a/fs/direct-io.c b/fs/direct-io.c index d53fa92a1ab6..2660e744da2d 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -434,7 +434,7 @@ static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) bio_set_pages_dirty(bio); - dio->bio_disk = bio->bi_disk; + dio->bio_disk = bio->bi_bdev->bd_disk; if (sdio->submit_io) { sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index aa34d620bec9..8cbf03159752 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -427,16 +427,6 @@ int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) return 0; } -/* - * Return true, if pre_bio's bdev is same as its target device. - */ -static bool __same_bdev(struct f2fs_sb_info *sbi, - block_t blk_addr, struct bio *bio) -{ - struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL); - return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno; -} - static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) { struct f2fs_sb_info *sbi = fio->sbi; @@ -741,7 +731,7 @@ static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, return false; if (last_blkaddr + 1 != cur_blkaddr) return false; - return __same_bdev(sbi, cur_blkaddr, bio); + return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL); } static bool io_type_is_mergeable(struct f2fs_bio_info *io, diff --git a/include/linux/bio.h b/include/linux/bio.h index 1edda614f7ce..12af7aa5db37 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -483,24 +483,22 @@ extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int); extern unsigned int bvec_nr_vecs(unsigned short idx); extern const char *bio_devname(struct bio *bio, char *buffer); -#define bio_set_dev(bio, bdev) \ -do { \ - if ((bio)->bi_disk != (bdev)->bd_disk) \ - bio_clear_flag(bio, BIO_THROTTLED);\ - (bio)->bi_disk = (bdev)->bd_disk; \ - (bio)->bi_partno = (bdev)->bd_partno; \ - bio_associate_blkg(bio); \ +#define bio_set_dev(bio, bdev) \ +do { \ + if ((bio)->bi_bdev != (bdev)) \ + bio_clear_flag(bio, BIO_THROTTLED); \ + (bio)->bi_bdev = (bdev); \ + bio_associate_blkg(bio); \ } while (0) #define bio_copy_dev(dst, src) \ do { \ - (dst)->bi_disk = (src)->bi_disk; \ - (dst)->bi_partno = (src)->bi_partno; \ + (dst)->bi_bdev = (src)->bi_bdev; \ bio_clone_blkg_association(dst, src); \ } while (0) #define bio_dev(bio) \ - disk_devt((bio)->bi_disk) + disk_devt((bio)->bi_bdev->bd_disk) #ifdef CONFIG_BLK_CGROUP void bio_associate_blkg(struct bio *bio); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index d705b174d346..6b410dab48ee 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -602,8 +602,8 @@ static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio, rq->bio = rq->biotail = bio; rq->ioprio = bio_prio(bio); - if (bio->bi_disk) - rq->rq_disk = bio->bi_disk; + if (bio->bi_bdev) + rq->rq_disk = bio->bi_bdev->bd_disk; } blk_qc_t blk_mq_submit_bio(struct bio *bio); diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 866f74261b3b..8ebd8be3e050 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -222,7 +222,7 @@ static inline void bio_issue_init(struct bio_issue *issue, */ struct bio { struct bio *bi_next; /* request queue link */ - struct gendisk *bi_disk; + struct block_device *bi_bdev; unsigned int bi_opf; /* bottom bits req flags, * top bits REQ_OP. Use * accessors. @@ -231,7 +231,6 @@ struct bio { unsigned short bi_ioprio; unsigned short bi_write_hint; blk_status_t bi_status; - u8 bi_partno; atomic_t __bi_remaining; struct bvec_iter bi_iter; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index f94ee3089e01..b55bd534b2e1 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1967,7 +1967,8 @@ void part_end_io_acct(struct block_device *part, struct bio *bio, */ static inline unsigned long bio_start_io_acct(struct bio *bio) { - return disk_start_io_acct(bio->bi_disk, bio_sectors(bio), bio_op(bio)); + return disk_start_io_acct(bio->bi_bdev->bd_disk, bio_sectors(bio), + bio_op(bio)); } /** @@ -1977,7 +1978,7 @@ static inline unsigned long bio_start_io_acct(struct bio *bio) */ static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time) { - return disk_end_io_acct(bio->bi_disk, bio_op(bio), start_time); + return disk_end_io_acct(bio->bi_bdev->bd_disk, bio_op(bio), start_time); } int bdev_read_only(struct block_device *bdev); diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index fb0fe4c66b84..9e9ee4945043 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -903,7 +903,7 @@ static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, static void blk_add_trace_bio_bounce(void *ignore, struct bio *bio) { - blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_BOUNCE, 0); + blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BOUNCE, 0); } static void blk_add_trace_bio_complete(void *ignore, @@ -915,22 +915,24 @@ static void blk_add_trace_bio_complete(void *ignore, static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio) { - blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_BACKMERGE, 0); + blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BACKMERGE, + 0); } static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio) { - blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_FRONTMERGE, 0); + blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_FRONTMERGE, + 0); } static void blk_add_trace_bio_queue(void *ignore, struct bio *bio) { - blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_QUEUE, 0); + blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_QUEUE, 0); } static void blk_add_trace_getrq(void *ignore, struct bio *bio) { - blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_GETRQ, 0); + blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_GETRQ, 0); } static void blk_add_trace_plug(void *ignore, struct request_queue *q) @@ -967,7 +969,7 @@ static void blk_add_trace_unplug(void *ignore, struct request_queue *q, static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) { - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; struct blk_trace *bt; rcu_read_lock(); @@ -997,7 +999,7 @@ static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev, sector_t from) { - struct request_queue *q = bio->bi_disk->queue; + struct request_queue *q = bio->bi_bdev->bd_disk->queue; struct blk_trace *bt; struct blk_io_trace_remap r; diff --git a/mm/page_io.c b/mm/page_io.c index 9bca17ecc4df..a75f35464a4e 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -433,7 +433,7 @@ int swap_readpage(struct page *page, bool synchronous) ret = -ENOMEM; goto out; } - disk = bio->bi_disk; + disk = bio->bi_bdev->bd_disk; /* * Keep this task valid during swap readpage because the oom killer may * attempt to access it in the page fault retry time check. -- cgit v1.2.3 From 30c5d3456c272f0de0d7e7eb9fc355fa64a5f649 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Jan 2021 11:02:36 +0100 Subject: block: do not reassig ->bi_bdev when partition remapping There is no good reason to reassign ->bi_bdev when remapping the partition-relative block number to the device wide one, as all the information required by the drivers comes from the gendisk anyway. Keeping the original ->bi_bdev alive will allow to greatly simplify the partition-away I/O accounting. Signed-off-by: Christoph Hellwig Acked-by: Tejun Heo Signed-off-by: Jens Axboe --- block/blk-core.c | 5 +++-- include/linux/bio.h | 2 ++ include/linux/blk_types.h | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/block/blk-core.c b/block/blk-core.c index 64f69022de96..1c1b97a82caa 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -752,7 +752,7 @@ static int blk_partition_remap(struct bio *bio) bio->bi_iter.bi_sector - p->bd_start_sect); } - bio->bi_bdev = bdev_whole(p); + bio_set_flag(bio, BIO_REMAPPED); return 0; } @@ -817,7 +817,8 @@ static noinline_for_stack bool submit_bio_checks(struct bio *bio) goto end_io; if (unlikely(bio_check_eod(bio))) goto end_io; - if (bio->bi_bdev->bd_partno && unlikely(blk_partition_remap(bio))) + if (bio->bi_bdev->bd_partno && !bio_flagged(bio, BIO_REMAPPED) && + unlikely(blk_partition_remap(bio))) goto end_io; /* diff --git a/include/linux/bio.h b/include/linux/bio.h index 12af7aa5db37..2f1155eabaff 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -485,6 +485,7 @@ extern const char *bio_devname(struct bio *bio, char *buffer); #define bio_set_dev(bio, bdev) \ do { \ + bio_clear_flag(bio, BIO_REMAPPED); \ if ((bio)->bi_bdev != (bdev)) \ bio_clear_flag(bio, BIO_THROTTLED); \ (bio)->bi_bdev = (bdev); \ @@ -493,6 +494,7 @@ do { \ #define bio_copy_dev(dst, src) \ do { \ + bio_clear_flag(dst, BIO_REMAPPED); \ (dst)->bi_bdev = (src)->bi_bdev; \ bio_clone_blkg_association(dst, src); \ } while (0) diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 8ebd8be3e050..1bc6f6a01070 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -303,6 +303,7 @@ enum { * of this bio. */ BIO_CGROUP_ACCT, /* has been accounted to a cgroup */ BIO_TRACKED, /* set if bio goes through the rq_qos path */ + BIO_REMAPPED, BIO_FLAG_LAST }; -- cgit v1.2.3 From 99dfc43ecbf67f12a06512918aaba61d55863efc Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Jan 2021 11:02:37 +0100 Subject: block: use ->bi_bdev for bio based I/O accounting Rework the I/O accounting for bio based drivers to use ->bi_bdev. This means all drivers can now simply use bio_start_io_acct to start accounting, and it will take partitions into account automatically. To end I/O account either bio_end_io_acct can be used if the driver never remaps I/O to a different device, or bio_end_io_acct_remapped if the driver did remap the I/O. Signed-off-by: Christoph Hellwig Acked-by: Tejun Heo Signed-off-by: Jens Axboe --- block/blk-core.c | 23 +++++++++++++---------- drivers/md/bcache/request.c | 34 +++++++++++++++++++++------------- drivers/md/md.c | 8 ++++---- include/linux/blkdev.h | 21 ++++----------------- 4 files changed, 42 insertions(+), 44 deletions(-) (limited to 'include/linux') diff --git a/block/blk-core.c b/block/blk-core.c index 1c1b97a82caa..9315311c27a9 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1321,14 +1321,17 @@ static unsigned long __part_start_io_acct(struct block_device *part, return now; } -unsigned long part_start_io_acct(struct gendisk *disk, struct block_device **part, - struct bio *bio) +/** + * bio_start_io_acct - start I/O accounting for bio based drivers + * @bio: bio to start account for + * + * Returns the start time that should be passed back to bio_end_io_acct(). + */ +unsigned long bio_start_io_acct(struct bio *bio) { - *part = disk_map_sector_rcu(disk, bio->bi_iter.bi_sector); - - return __part_start_io_acct(*part, bio_sectors(bio), bio_op(bio)); + return __part_start_io_acct(bio->bi_bdev, bio_sectors(bio), bio_op(bio)); } -EXPORT_SYMBOL_GPL(part_start_io_acct); +EXPORT_SYMBOL_GPL(bio_start_io_acct); unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, unsigned int op) @@ -1351,12 +1354,12 @@ static void __part_end_io_acct(struct block_device *part, unsigned int op, part_stat_unlock(); } -void part_end_io_acct(struct block_device *part, struct bio *bio, - unsigned long start_time) +void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time, + struct block_device *orig_bdev) { - __part_end_io_acct(part, bio_op(bio), start_time); + __part_end_io_acct(orig_bdev, bio_op(bio), start_time); } -EXPORT_SYMBOL_GPL(part_end_io_acct); +EXPORT_SYMBOL_GPL(bio_end_io_acct_remapped); void disk_end_io_acct(struct gendisk *disk, unsigned int op, unsigned long start_time) diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index dfc35d6d05ed..29c231758293 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -475,7 +475,7 @@ struct search { unsigned int read_dirty_data:1; unsigned int cache_missed:1; - struct block_device *part; + struct block_device *orig_bdev; unsigned long start_time; struct btree_op op; @@ -670,8 +670,8 @@ static void bio_complete(struct search *s) { if (s->orig_bio) { /* Count on bcache device */ - part_end_io_acct(s->part, s->orig_bio, s->start_time); - + bio_end_io_acct_remapped(s->orig_bio, s->start_time, + s->orig_bdev); trace_bcache_request_end(s->d, s->orig_bio); s->orig_bio->bi_status = s->iop.status; bio_endio(s->orig_bio); @@ -714,7 +714,8 @@ static void search_free(struct closure *cl) } static inline struct search *search_alloc(struct bio *bio, - struct bcache_device *d) + struct bcache_device *d, struct block_device *orig_bdev, + unsigned long start_time) { struct search *s; @@ -732,7 +733,8 @@ static inline struct search *search_alloc(struct bio *bio, s->write = op_is_write(bio_op(bio)); s->read_dirty_data = 0; /* Count on the bcache device */ - s->start_time = part_start_io_acct(d->disk, &s->part, bio); + s->orig_bdev = orig_bdev; + s->start_time = start_time; s->iop.c = d->c; s->iop.bio = NULL; s->iop.inode = d->id; @@ -1074,7 +1076,7 @@ struct detached_dev_io_private { unsigned long start_time; bio_end_io_t *bi_end_io; void *bi_private; - struct block_device *part; + struct block_device *orig_bdev; }; static void detached_dev_end_io(struct bio *bio) @@ -1086,7 +1088,7 @@ static void detached_dev_end_io(struct bio *bio) bio->bi_private = ddip->bi_private; /* Count on the bcache device */ - part_end_io_acct(ddip->part, bio, ddip->start_time); + bio_end_io_acct_remapped(bio, ddip->start_time, ddip->orig_bdev); if (bio->bi_status) { struct cached_dev *dc = container_of(ddip->d, @@ -1099,7 +1101,8 @@ static void detached_dev_end_io(struct bio *bio) bio->bi_end_io(bio); } -static void detached_dev_do_request(struct bcache_device *d, struct bio *bio) +static void detached_dev_do_request(struct bcache_device *d, struct bio *bio, + struct block_device *orig_bdev, unsigned long start_time) { struct detached_dev_io_private *ddip; struct cached_dev *dc = container_of(d, struct cached_dev, disk); @@ -1112,7 +1115,8 @@ static void detached_dev_do_request(struct bcache_device *d, struct bio *bio) ddip = kzalloc(sizeof(struct detached_dev_io_private), GFP_NOIO); ddip->d = d; /* Count on the bcache device */ - ddip->start_time = part_start_io_acct(d->disk, &ddip->part, bio); + ddip->orig_bdev = orig_bdev; + ddip->start_time = start_time; ddip->bi_end_io = bio->bi_end_io; ddip->bi_private = bio->bi_private; bio->bi_end_io = detached_dev_end_io; @@ -1168,8 +1172,10 @@ static void quit_max_writeback_rate(struct cache_set *c, blk_qc_t cached_dev_submit_bio(struct bio *bio) { struct search *s; - struct bcache_device *d = bio->bi_bdev->bd_disk->private_data; + struct block_device *orig_bdev = bio->bi_bdev; + struct bcache_device *d = orig_bdev->bd_disk->private_data; struct cached_dev *dc = container_of(d, struct cached_dev, disk); + unsigned long start_time; int rw = bio_data_dir(bio); if (unlikely((d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags)) || @@ -1194,11 +1200,13 @@ blk_qc_t cached_dev_submit_bio(struct bio *bio) } } + start_time = bio_start_io_acct(bio); + bio_set_dev(bio, dc->bdev); bio->bi_iter.bi_sector += dc->sb.data_offset; if (cached_dev_get(dc)) { - s = search_alloc(bio, d); + s = search_alloc(bio, d, orig_bdev, start_time); trace_bcache_request_start(s->d, bio); if (!bio->bi_iter.bi_size) { @@ -1219,7 +1227,7 @@ blk_qc_t cached_dev_submit_bio(struct bio *bio) } } else /* I/O request sent to backing device */ - detached_dev_do_request(d, bio); + detached_dev_do_request(d, bio, orig_bdev, start_time); return BLK_QC_T_NONE; } @@ -1283,7 +1291,7 @@ blk_qc_t flash_dev_submit_bio(struct bio *bio) return BLK_QC_T_NONE; } - s = search_alloc(bio, d); + s = search_alloc(bio, d, bio->bi_bdev, bio_start_io_acct(bio)); cl = &s->cl; bio = &s->bio.bio; diff --git a/drivers/md/md.c b/drivers/md/md.c index cf06dbb1aa53..7d1bb24add31 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -463,8 +463,8 @@ struct md_io { struct mddev *mddev; bio_end_io_t *orig_bi_end_io; void *orig_bi_private; + struct block_device *orig_bi_bdev; unsigned long start_time; - struct block_device *part; }; static void md_end_io(struct bio *bio) @@ -472,7 +472,7 @@ static void md_end_io(struct bio *bio) struct md_io *md_io = bio->bi_private; struct mddev *mddev = md_io->mddev; - part_end_io_acct(md_io->part, bio, md_io->start_time); + bio_end_io_acct_remapped(bio, md_io->start_time, md_io->orig_bi_bdev); bio->bi_end_io = md_io->orig_bi_end_io; bio->bi_private = md_io->orig_bi_private; @@ -514,12 +514,12 @@ static blk_qc_t md_submit_bio(struct bio *bio) md_io->mddev = mddev; md_io->orig_bi_end_io = bio->bi_end_io; md_io->orig_bi_private = bio->bi_private; + md_io->orig_bi_bdev = bio->bi_bdev; bio->bi_end_io = md_end_io; bio->bi_private = md_io; - md_io->start_time = part_start_io_acct(mddev->gendisk, - &md_io->part, bio); + md_io->start_time = bio_start_io_acct(bio); } /* bio could be mergeable after passing to underlayer */ diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b55bd534b2e1..4526b9ef8edb 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1954,22 +1954,9 @@ unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors, void disk_end_io_acct(struct gendisk *disk, unsigned int op, unsigned long start_time); -unsigned long part_start_io_acct(struct gendisk *disk, - struct block_device **part, struct bio *bio); -void part_end_io_acct(struct block_device *part, struct bio *bio, - unsigned long start_time); - -/** - * bio_start_io_acct - start I/O accounting for bio based drivers - * @bio: bio to start account for - * - * Returns the start time that should be passed back to bio_end_io_acct(). - */ -static inline unsigned long bio_start_io_acct(struct bio *bio) -{ - return disk_start_io_acct(bio->bi_bdev->bd_disk, bio_sectors(bio), - bio_op(bio)); -} +unsigned long bio_start_io_acct(struct bio *bio); +void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time, + struct block_device *orig_bdev); /** * bio_end_io_acct - end I/O accounting for bio based drivers @@ -1978,7 +1965,7 @@ static inline unsigned long bio_start_io_acct(struct bio *bio) */ static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time) { - return disk_end_io_acct(bio->bi_bdev->bd_disk, bio_op(bio), start_time); + return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev); } int bdev_read_only(struct block_device *bdev); -- cgit v1.2.3 From bc359d03c7ec1bf3b86d03bafaf6bbb21e6414fd Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Jan 2021 11:02:39 +0100 Subject: block: add a disk_uevent helper Add a helper to call kobject_uevent for the disk and all partitions, and unexport the disk_part_iter_* helpers that are now only used in the core block code. Signed-off-by: Christoph Hellwig Acked-by: Tejun Heo Signed-off-by: Jens Axboe --- block/genhd.c | 27 ++++++++++++++------------- drivers/s390/block/dasd.c | 26 +++++--------------------- include/linux/genhd.h | 2 ++ 3 files changed, 21 insertions(+), 34 deletions(-) (limited to 'include/linux') diff --git a/block/genhd.c b/block/genhd.c index e46de616a19e..7094612c7510 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -203,7 +203,6 @@ void disk_part_iter_init(struct disk_part_iter *piter, struct gendisk *disk, rcu_read_unlock(); } -EXPORT_SYMBOL_GPL(disk_part_iter_init); /** * disk_part_iter_next - proceed iterator to the next partition and return it @@ -266,7 +265,6 @@ struct block_device *disk_part_iter_next(struct disk_part_iter *piter) return piter->part; } -EXPORT_SYMBOL_GPL(disk_part_iter_next); /** * disk_part_iter_exit - finish up partition iteration @@ -283,7 +281,6 @@ void disk_part_iter_exit(struct disk_part_iter *piter) bdput(piter->part); piter->part = NULL; } -EXPORT_SYMBOL_GPL(disk_part_iter_exit); /** * disk_has_partitions @@ -555,6 +552,18 @@ static char *bdevt_str(dev_t devt, char *buf) return buf; } +void disk_uevent(struct gendisk *disk, enum kobject_action action) +{ + struct disk_part_iter piter; + struct block_device *part; + + disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0); + while ((part = disk_part_iter_next(&piter))) + kobject_uevent(bdev_kobj(part), action); + disk_part_iter_exit(&piter); +} +EXPORT_SYMBOL_GPL(disk_uevent); + static void disk_scan_partitions(struct gendisk *disk) { struct block_device *bdev; @@ -572,8 +581,6 @@ static void register_disk(struct device *parent, struct gendisk *disk, const struct attribute_group **groups) { struct device *ddev = disk_to_dev(disk); - struct disk_part_iter piter; - struct block_device *part; int err; ddev->parent = parent; @@ -616,15 +623,9 @@ static void register_disk(struct device *parent, struct gendisk *disk, disk_scan_partitions(disk); - /* announce disk after possible partitions are created */ + /* announce the disk and partitions after all partitions are created */ dev_set_uevent_suppress(ddev, 0); - kobject_uevent(&ddev->kobj, KOBJ_ADD); - - /* announce possible partitions */ - disk_part_iter_init(&piter, disk, 0); - while ((part = disk_part_iter_next(&piter))) - kobject_uevent(bdev_kobj(part), KOBJ_ADD); - disk_part_iter_exit(&piter); + disk_uevent(disk, KOBJ_ADD); if (disk->queue->backing_dev_info->dev) { err = sysfs_create_link(&ddev->kobj, diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index c7eb9a10c680..28c04a4efa66 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -428,23 +428,15 @@ static int dasd_state_unfmt_to_basic(struct dasd_device *device) static int dasd_state_ready_to_online(struct dasd_device * device) { - struct gendisk *disk; - struct disk_part_iter piter; - struct block_device *part; - device->state = DASD_STATE_ONLINE; if (device->block) { dasd_schedule_block_bh(device->block); if ((device->features & DASD_FEATURE_USERAW)) { - disk = device->block->gdp; - kobject_uevent(&disk_to_dev(disk)->kobj, KOBJ_CHANGE); + kobject_uevent(&disk_to_dev(device->block->gdp)->kobj, + KOBJ_CHANGE); return 0; } - disk = device->block->bdev->bd_disk; - disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0); - while ((part = disk_part_iter_next(&piter))) - kobject_uevent(bdev_kobj(part), KOBJ_CHANGE); - disk_part_iter_exit(&piter); + disk_uevent(device->block->bdev->bd_disk, KOBJ_CHANGE); } return 0; } @@ -455,9 +447,6 @@ dasd_state_ready_to_online(struct dasd_device * device) static int dasd_state_online_to_ready(struct dasd_device *device) { int rc; - struct gendisk *disk; - struct disk_part_iter piter; - struct block_device *part; if (device->discipline->online_to_ready) { rc = device->discipline->online_to_ready(device); @@ -466,13 +455,8 @@ static int dasd_state_online_to_ready(struct dasd_device *device) } device->state = DASD_STATE_READY; - if (device->block && !(device->features & DASD_FEATURE_USERAW)) { - disk = device->block->bdev->bd_disk; - disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0); - while ((part = disk_part_iter_next(&piter))) - kobject_uevent(bdev_kobj(part), KOBJ_CHANGE); - disk_part_iter_exit(&piter); - } + if (device->block && !(device->features & DASD_FEATURE_USERAW)) + disk_uevent(device->block->bdev->bd_disk, KOBJ_CHANGE); return 0; } diff --git a/include/linux/genhd.h b/include/linux/genhd.h index a62ccbfac54b..670eaef0e876 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -213,6 +213,8 @@ static inline dev_t disk_devt(struct gendisk *disk) return MKDEV(disk->major, disk->first_minor); } +void disk_uevent(struct gendisk *disk, enum kobject_action action); + /* * Smarter partition iterator without context limits. */ -- cgit v1.2.3 From 0470dd9d5f103e7f1d5ba8f755f687c3106c7df1 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Jan 2021 11:02:40 +0100 Subject: block: remove DISK_PITER_REVERSE There is good reason to iterate backwards when deleting all partitions in del_gendisk, just like we don't in blk_drop_partitions. Signed-off-by: Christoph Hellwig Acked-by: Tejun Heo Signed-off-by: Jens Axboe --- block/genhd.c | 37 +++++++------------------------------ include/linux/genhd.h | 1 - 2 files changed, 7 insertions(+), 31 deletions(-) (limited to 'include/linux') diff --git a/block/genhd.c b/block/genhd.c index 7094612c7510..1832add5c738 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -184,24 +184,13 @@ static struct block_device *__disk_get_part(struct gendisk *disk, int partno) void disk_part_iter_init(struct disk_part_iter *piter, struct gendisk *disk, unsigned int flags) { - struct disk_part_tbl *ptbl; - - rcu_read_lock(); - ptbl = rcu_dereference(disk->part_tbl); - piter->disk = disk; piter->part = NULL; - - if (flags & DISK_PITER_REVERSE) - piter->idx = ptbl->len - 1; - else if (flags & (DISK_PITER_INCL_PART0 | DISK_PITER_INCL_EMPTY_PART0)) + if (flags & (DISK_PITER_INCL_PART0 | DISK_PITER_INCL_EMPTY_PART0)) piter->idx = 0; else piter->idx = 1; - piter->flags = flags; - - rcu_read_unlock(); } /** @@ -216,7 +205,6 @@ void disk_part_iter_init(struct disk_part_iter *piter, struct gendisk *disk, struct block_device *disk_part_iter_next(struct disk_part_iter *piter) { struct disk_part_tbl *ptbl; - int inc, end; /* put the last partition */ disk_part_iter_exit(piter); @@ -225,21 +213,8 @@ struct block_device *disk_part_iter_next(struct disk_part_iter *piter) rcu_read_lock(); ptbl = rcu_dereference(piter->disk->part_tbl); - /* determine iteration parameters */ - if (piter->flags & DISK_PITER_REVERSE) { - inc = -1; - if (piter->flags & (DISK_PITER_INCL_PART0 | - DISK_PITER_INCL_EMPTY_PART0)) - end = -1; - else - end = 0; - } else { - inc = 1; - end = ptbl->len; - } - /* iterate to the next partition */ - for (; piter->idx != end; piter->idx += inc) { + for (; piter->idx != ptbl->len; piter->idx += 1) { struct block_device *part; part = rcu_dereference(ptbl->part[piter->idx]); @@ -257,7 +232,10 @@ struct block_device *disk_part_iter_next(struct disk_part_iter *piter) continue; } - piter->idx += inc; + piter->part = bdgrab(part); + if (!piter->part) + continue; + piter->idx += 1; break; } @@ -781,8 +759,7 @@ void del_gendisk(struct gendisk *disk) down_write(&bdev_lookup_sem); /* invalidate stuff */ - disk_part_iter_init(&piter, disk, - DISK_PITER_INCL_EMPTY | DISK_PITER_REVERSE); + disk_part_iter_init(&piter, disk, DISK_PITER_INCL_EMPTY); while ((part = disk_part_iter_next(&piter))) { invalidate_partition(part); delete_partition(part); diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 670eaef0e876..51609133c9a3 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -218,7 +218,6 @@ void disk_uevent(struct gendisk *disk, enum kobject_action action); /* * Smarter partition iterator without context limits. */ -#define DISK_PITER_REVERSE (1 << 0) /* iterate in the reverse direction */ #define DISK_PITER_INCL_EMPTY (1 << 1) /* include 0-sized parts */ #define DISK_PITER_INCL_PART0 (1 << 2) /* include partition 0 */ #define DISK_PITER_INCL_EMPTY_PART0 (1 << 3) /* include empty partition 0 */ -- cgit v1.2.3 From a33df75c6328bf40078b35f2040d8e54d574c357 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Jan 2021 11:02:41 +0100 Subject: block: use an xarray for disk->part_tbl Now that no fast path lookups in the partition table are left, there is no point in micro-optimizing the data structure for it. Just use a bog standard xarray. Signed-off-by: Christoph Hellwig Acked-by: Tejun Heo Signed-off-by: Jens Axboe --- block/blk-settings.c | 2 +- block/blk.h | 1 - block/genhd.c | 163 ++++-------------------------------------------- block/partitions/core.c | 31 ++------- include/linux/genhd.h | 18 +----- 5 files changed, 22 insertions(+), 193 deletions(-) (limited to 'include/linux') diff --git a/block/blk-settings.c b/block/blk-settings.c index 43990b1d148b..4c974340f1a9 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -865,7 +865,7 @@ void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model) * we do nothing special as far as the block layer is concerned. */ if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED) || - disk_has_partitions(disk)) + !xa_empty(&disk->part_tbl)) model = BLK_ZONED_NONE; break; case BLK_ZONED_NONE: diff --git a/block/blk.h b/block/blk.h index d965cacc5bda..ab0aaf958553 100644 --- a/block/blk.h +++ b/block/blk.h @@ -345,7 +345,6 @@ int bdev_add_partition(struct block_device *bdev, int partno, int bdev_del_partition(struct block_device *bdev, int partno); int bdev_resize_partition(struct block_device *bdev, int partno, sector_t start, sector_t length); -int disk_expand_part_tbl(struct gendisk *disk, int target); int bio_add_hw_page(struct request_queue *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset, diff --git a/block/genhd.c b/block/genhd.c index 1832add5c738..d3ef29fbc536 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -161,15 +161,6 @@ static void part_in_flight_rw(struct block_device *part, inflight[1] = 0; } -static struct block_device *__disk_get_part(struct gendisk *disk, int partno) -{ - struct disk_part_tbl *ptbl = rcu_dereference(disk->part_tbl); - - if (unlikely(partno < 0 || partno >= ptbl->len)) - return NULL; - return rcu_dereference(ptbl->part[partno]); -} - /** * disk_part_iter_init - initialize partition iterator * @piter: iterator to initialize @@ -204,41 +195,26 @@ void disk_part_iter_init(struct disk_part_iter *piter, struct gendisk *disk, */ struct block_device *disk_part_iter_next(struct disk_part_iter *piter) { - struct disk_part_tbl *ptbl; + struct block_device *part; + unsigned long idx; /* put the last partition */ disk_part_iter_exit(piter); - /* get part_tbl */ rcu_read_lock(); - ptbl = rcu_dereference(piter->disk->part_tbl); - - /* iterate to the next partition */ - for (; piter->idx != ptbl->len; piter->idx += 1) { - struct block_device *part; - - part = rcu_dereference(ptbl->part[piter->idx]); - if (!part) - continue; - piter->part = bdgrab(part); - if (!piter->part) - continue; + xa_for_each_start(&piter->disk->part_tbl, idx, part, piter->idx) { if (!bdev_nr_sectors(part) && !(piter->flags & DISK_PITER_INCL_EMPTY) && !(piter->flags & DISK_PITER_INCL_EMPTY_PART0 && - piter->idx == 0)) { - bdput(piter->part); - piter->part = NULL; + piter->idx == 0)) continue; - } piter->part = bdgrab(part); if (!piter->part) continue; - piter->idx += 1; + piter->idx = idx + 1; break; } - rcu_read_unlock(); return piter->part; @@ -260,42 +236,6 @@ void disk_part_iter_exit(struct disk_part_iter *piter) piter->part = NULL; } -/** - * disk_has_partitions - * @disk: gendisk of interest - * - * Walk through the partition table and check if valid partition exists. - * - * CONTEXT: - * Don't care. - * - * RETURNS: - * True if the gendisk has at least one valid non-zero size partition. - * Otherwise false. - */ -bool disk_has_partitions(struct gendisk *disk) -{ - struct disk_part_tbl *ptbl; - int i; - bool ret = false; - - rcu_read_lock(); - ptbl = rcu_dereference(disk->part_tbl); - - /* Iterate partitions skipping the whole device at index 0 */ - for (i = 1; i < ptbl->len; i++) { - if (rcu_dereference(ptbl->part[i])) { - ret = true; - break; - } - } - - rcu_read_unlock(); - - return ret; -} -EXPORT_SYMBOL_GPL(disk_has_partitions); - /* * Can be deleted altogether. Later. * @@ -858,7 +798,7 @@ struct block_device *bdget_disk(struct gendisk *disk, int partno) struct block_device *bdev = NULL; rcu_read_lock(); - bdev = __disk_get_part(disk, partno); + bdev = xa_load(&disk->part_tbl, partno); if (bdev && !bdgrab(bdev)) bdev = NULL; rcu_read_unlock(); @@ -1248,83 +1188,6 @@ static const struct attribute_group *disk_attr_groups[] = { NULL }; -/** - * disk_replace_part_tbl - replace disk->part_tbl in RCU-safe way - * @disk: disk to replace part_tbl for - * @new_ptbl: new part_tbl to install - * - * Replace disk->part_tbl with @new_ptbl in RCU-safe way. The - * original ptbl is freed using RCU callback. - * - * LOCKING: - * Matching bd_mutex locked or the caller is the only user of @disk. - */ -static void disk_replace_part_tbl(struct gendisk *disk, - struct disk_part_tbl *new_ptbl) -{ - struct disk_part_tbl *old_ptbl = - rcu_dereference_protected(disk->part_tbl, 1); - - rcu_assign_pointer(disk->part_tbl, new_ptbl); - - if (old_ptbl) { - rcu_assign_pointer(old_ptbl->last_lookup, NULL); - kfree_rcu(old_ptbl, rcu_head); - } -} - -/** - * disk_expand_part_tbl - expand disk->part_tbl - * @disk: disk to expand part_tbl for - * @partno: expand such that this partno can fit in - * - * Expand disk->part_tbl such that @partno can fit in. disk->part_tbl - * uses RCU to allow unlocked dereferencing for stats and other stuff. - * - * LOCKING: - * Matching bd_mutex locked or the caller is the only user of @disk. - * Might sleep. - * - * RETURNS: - * 0 on success, -errno on failure. - */ -int disk_expand_part_tbl(struct gendisk *disk, int partno) -{ - struct disk_part_tbl *old_ptbl = - rcu_dereference_protected(disk->part_tbl, 1); - struct disk_part_tbl *new_ptbl; - int len = old_ptbl ? old_ptbl->len : 0; - int i, target; - - /* - * check for int overflow, since we can get here from blkpg_ioctl() - * with a user passed 'partno'. - */ - target = partno + 1; - if (target < 0) - return -EINVAL; - - /* disk_max_parts() is zero during initialization, ignore if so */ - if (disk_max_parts(disk) && target > disk_max_parts(disk)) - return -EINVAL; - - if (target <= len) - return 0; - - new_ptbl = kzalloc_node(struct_size(new_ptbl, part, target), GFP_KERNEL, - disk->node_id); - if (!new_ptbl) - return -ENOMEM; - - new_ptbl->len = target; - - for (i = 0; i < len; i++) - rcu_assign_pointer(new_ptbl->part[i], old_ptbl->part[i]); - - disk_replace_part_tbl(disk, new_ptbl); - return 0; -} - /** * disk_release - releases all allocated resources of the gendisk * @dev: the device representing this disk @@ -1348,7 +1211,7 @@ static void disk_release(struct device *dev) blk_free_devt(dev->devt); disk_release_events(disk); kfree(disk->random); - disk_replace_part_tbl(disk, NULL); + xa_destroy(&disk->part_tbl); bdput(disk->part0); if (disk->queue) blk_put_queue(disk->queue); @@ -1501,7 +1364,6 @@ dev_t blk_lookup_devt(const char *name, int partno) struct gendisk *__alloc_disk_node(int minors, int node_id) { struct gendisk *disk; - struct disk_part_tbl *ptbl; if (minors > DISK_MAX_PARTS) { printk(KERN_ERR @@ -1519,11 +1381,9 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) goto out_free_disk; disk->node_id = node_id; - if (disk_expand_part_tbl(disk, 0)) - goto out_bdput; - - ptbl = rcu_dereference_protected(disk->part_tbl, 1); - rcu_assign_pointer(ptbl->part[0], disk->part0); + xa_init(&disk->part_tbl); + if (xa_insert(&disk->part_tbl, 0, disk->part0, GFP_KERNEL)) + goto out_destroy_part_tbl; disk->minors = minors; rand_initialize_disk(disk); @@ -1532,7 +1392,8 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) device_initialize(disk_to_dev(disk)); return disk; -out_bdput: +out_destroy_part_tbl: + xa_destroy(&disk->part_tbl); bdput(disk->part0); out_free_disk: kfree(disk); diff --git a/block/partitions/core.c b/block/partitions/core.c index 168d5906077c..b1cdf88f96e2 100644 --- a/block/partitions/core.c +++ b/block/partitions/core.c @@ -287,13 +287,7 @@ struct device_type part_type = { */ void delete_partition(struct block_device *part) { - struct gendisk *disk = part->bd_disk; - struct disk_part_tbl *ptbl = - rcu_dereference_protected(disk->part_tbl, 1); - - rcu_assign_pointer(ptbl->part[part->bd_partno], NULL); - rcu_assign_pointer(ptbl->last_lookup, NULL); - + xa_erase(&part->bd_disk->part_tbl, part->bd_partno); kobject_put(part->bd_holder_dir); device_del(&part->bd_device); @@ -325,7 +319,6 @@ static struct block_device *add_partition(struct gendisk *disk, int partno, struct device *ddev = disk_to_dev(disk); struct device *pdev; struct block_device *bdev; - struct disk_part_tbl *ptbl; const char *dname; int err; @@ -347,12 +340,7 @@ static struct block_device *add_partition(struct gendisk *disk, int partno, break; } - err = disk_expand_part_tbl(disk, partno); - if (err) - return ERR_PTR(err); - ptbl = rcu_dereference_protected(disk->part_tbl, 1); - - if (ptbl->part[partno]) + if (xa_load(&disk->part_tbl, partno)) return ERR_PTR(-EBUSY); bdev = bdev_alloc(disk, partno); @@ -405,8 +393,10 @@ static struct block_device *add_partition(struct gendisk *disk, int partno, } /* everything is up and running, commence */ + err = xa_insert(&disk->part_tbl, partno, bdev, GFP_KERNEL); + if (err) + goto out_del; bdev_add(bdev, devt); - rcu_assign_pointer(ptbl->part[partno], bdev); /* suppress uevent if the disk suppresses it */ if (!dev_get_uevent_suppress(ddev)) @@ -612,7 +602,7 @@ static bool blk_add_partition(struct gendisk *disk, struct block_device *bdev, int blk_add_partitions(struct gendisk *disk, struct block_device *bdev) { struct parsed_partitions *state; - int ret = -EAGAIN, p, highest; + int ret = -EAGAIN, p; if (!disk_part_scan_enabled(disk)) return 0; @@ -660,15 +650,6 @@ int blk_add_partitions(struct gendisk *disk, struct block_device *bdev) /* tell userspace that the media / partition table may have changed */ kobject_uevent(&disk_to_dev(disk)->kobj, KOBJ_CHANGE); - /* - * Detect the highest partition number and preallocate disk->part_tbl. - * This is an optimization and not strictly necessary. - */ - for (p = 1, highest = 0; p < state->limit; p++) - if (state->parts[p].size) - highest = p; - disk_expand_part_tbl(disk, highest); - for (p = 1; p < state->limit; p++) if (!blk_add_partition(disk, bdev, state, p)) goto out_free_state; diff --git a/include/linux/genhd.h b/include/linux/genhd.h index 51609133c9a3..f364619092cc 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -32,6 +32,7 @@ extern struct class block_class; #include #include #include +#include #define PARTITION_META_INFO_VOLNAMELTH 64 /* @@ -116,13 +117,6 @@ enum { DISK_EVENT_FLAG_UEVENT = 1 << 1, }; -struct disk_part_tbl { - struct rcu_head rcu_head; - int len; - struct block_device __rcu *last_lookup; - struct block_device __rcu *part[]; -}; - struct disk_events; struct badblocks; @@ -148,12 +142,7 @@ struct gendisk { unsigned short events; /* supported events */ unsigned short event_flags; /* flags related to event processing */ - /* Array of pointers to partitions indexed by partno. - * Protected with matching bdev lock but stat and other - * non-critical accesses use RCU. Always access through - * helpers. - */ - struct disk_part_tbl __rcu *part_tbl; + struct xarray part_tbl; struct block_device *part0; const struct block_device_operations *fops; @@ -225,7 +214,7 @@ void disk_uevent(struct gendisk *disk, enum kobject_action action); struct disk_part_iter { struct gendisk *disk; struct block_device *part; - int idx; + unsigned long idx; unsigned int flags; }; @@ -233,7 +222,6 @@ extern void disk_part_iter_init(struct disk_part_iter *piter, struct gendisk *disk, unsigned int flags); struct block_device *disk_part_iter_next(struct disk_part_iter *piter); extern void disk_part_iter_exit(struct disk_part_iter *piter); -extern bool disk_has_partitions(struct gendisk *disk); /* block/genhd.c */ extern void device_add_disk(struct device *parent, struct gendisk *disk, -- cgit v1.2.3 From 5ac83c644f5fb924f0b2c09102ab82fc788f8411 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 11 Jan 2021 17:47:16 +0100 Subject: Revert "blk-mq, elevator: Count requests per hctx to improve performance" This reverts commit b445547ec1bbd3e7bf4b1c142550942f70527d95. Since both mq-deadline and BFQ completely ignore hctx they are passed to their dispatch function and dispatch whatever request they deem fit checking whether any request for a particular hctx is queued is just pointless since we'll very likely get a request from a different hctx anyway. In the following commit we'll deal with lock contention in these IO schedulers in presence of multiple HW queues in a different way. Signed-off-by: Jan Kara Reviewed-by: Ming Lei Signed-off-by: Jens Axboe --- block/bfq-iosched.c | 5 ----- block/blk-mq.c | 1 - block/mq-deadline.c | 6 ------ include/linux/blk-mq.h | 4 ---- 4 files changed, 16 deletions(-) (limited to 'include/linux') diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index c045613ce927..b12a416b51d7 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -4677,9 +4677,6 @@ static bool bfq_has_work(struct blk_mq_hw_ctx *hctx) { struct bfq_data *bfqd = hctx->queue->elevator->elevator_data; - if (!atomic_read(&hctx->elevator_queued)) - return false; - /* * Avoiding lock: a race on bfqd->busy_queues should cause at * most a call to dispatch for nothing @@ -5597,7 +5594,6 @@ static void bfq_insert_requests(struct blk_mq_hw_ctx *hctx, rq = list_first_entry(list, struct request, queuelist); list_del_init(&rq->queuelist); bfq_insert_request(hctx, rq, at_head); - atomic_inc(&hctx->elevator_queued); } } @@ -5965,7 +5961,6 @@ static void bfq_finish_requeue_request(struct request *rq) bfq_completed_request(bfqq, bfqd); bfq_finish_requeue_request_body(bfqq); - atomic_dec(&rq->mq_hctx->elevator_queued); spin_unlock_irqrestore(&bfqd->lock, flags); } else { diff --git a/block/blk-mq.c b/block/blk-mq.c index 74b17b396f4c..1af6b8a9da5a 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2653,7 +2653,6 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set, goto free_hctx; atomic_set(&hctx->nr_active, 0); - atomic_set(&hctx->elevator_queued, 0); if (node == NUMA_NO_NODE) node = set->numa_node; hctx->numa_node = node; diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 800ac902809b..b57470e154c8 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -386,8 +386,6 @@ static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) spin_lock(&dd->lock); rq = __dd_dispatch_request(dd); spin_unlock(&dd->lock); - if (rq) - atomic_dec(&rq->mq_hctx->elevator_queued); return rq; } @@ -535,7 +533,6 @@ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx, rq = list_first_entry(list, struct request, queuelist); list_del_init(&rq->queuelist); dd_insert_request(hctx, rq, at_head); - atomic_inc(&hctx->elevator_queued); } spin_unlock(&dd->lock); } @@ -582,9 +579,6 @@ static bool dd_has_work(struct blk_mq_hw_ctx *hctx) { struct deadline_data *dd = hctx->queue->elevator->elevator_data; - if (!atomic_read(&hctx->elevator_queued)) - return false; - return !list_empty_careful(&dd->dispatch) || !list_empty_careful(&dd->fifo_list[0]) || !list_empty_careful(&dd->fifo_list[1]); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 6b410dab48ee..aabbf6830ffc 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -140,10 +140,6 @@ struct blk_mq_hw_ctx { * shared across request queues. */ atomic_t nr_active; - /** - * @elevator_queued: Number of queued requests on hctx. - */ - atomic_t elevator_queued; /** @cpuhp_online: List to store request if CPU is going to die */ struct hlist_node cpuhp_online; -- cgit v1.2.3 From b6e68ee82585f2ee890b0a897a6aacbf49a467bb Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 11 Jan 2021 17:47:17 +0100 Subject: blk-mq: Improve performance of non-mq IO schedulers with multiple HW queues Currently when non-mq aware IO scheduler (BFQ, mq-deadline) is used for a queue with multiple HW queues, the performance it rather bad. The problem is that these IO schedulers use queue-wide locking and their dispatch function does not respect the hctx it is passed in and returns any request it finds appropriate. Thus locality of request access is broken and dispatch from multiple CPUs just contends on IO scheduler locks. For these IO schedulers there's little point in dispatching from multiple CPUs. Instead dispatch always only from a single CPU to limit contention. Below is a comparison of dbench runs on XFS filesystem where the storage is a raid card with 64 HW queues and to it attached a single rotating disk. BFQ is used as IO scheduler: clients MQ SQ MQ-Patched Amean 1 39.12 (0.00%) 43.29 * -10.67%* 36.09 * 7.74%* Amean 2 128.58 (0.00%) 101.30 * 21.22%* 96.14 * 25.23%* Amean 4 577.42 (0.00%) 494.47 * 14.37%* 508.49 * 11.94%* Amean 8 610.95 (0.00%) 363.86 * 40.44%* 362.12 * 40.73%* Amean 16 391.78 (0.00%) 261.49 * 33.25%* 282.94 * 27.78%* Amean 32 324.64 (0.00%) 267.71 * 17.54%* 233.00 * 28.23%* Amean 64 295.04 (0.00%) 253.02 * 14.24%* 242.37 * 17.85%* Amean 512 10281.61 (0.00%) 10211.16 * 0.69%* 10447.53 * -1.61%* Numbers are times so lower is better. MQ is stock 5.10-rc6 kernel. SQ is the same kernel with megaraid_sas.host_tagset_enable=0 so that the card advertises just a single HW queue. MQ-Patched is a kernel with this patch applied. You can see multiple hardware queues heavily hurt performance in combination with BFQ. The patch restores the performance. Signed-off-by: Jan Kara Reviewed-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-mq.c | 66 +++++++++++++++++++++++++++++++++++++++++++----- block/kyber-iosched.c | 1 + include/linux/elevator.h | 2 ++ 3 files changed, 63 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/block/blk-mq.c b/block/blk-mq.c index 1af6b8a9da5a..f21d922ecfaf 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1646,6 +1646,42 @@ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) } EXPORT_SYMBOL(blk_mq_run_hw_queue); +/* + * Is the request queue handled by an IO scheduler that does not respect + * hardware queues when dispatching? + */ +static bool blk_mq_has_sqsched(struct request_queue *q) +{ + struct elevator_queue *e = q->elevator; + + if (e && e->type->ops.dispatch_request && + !(e->type->elevator_features & ELEVATOR_F_MQ_AWARE)) + return true; + return false; +} + +/* + * Return prefered queue to dispatch from (if any) for non-mq aware IO + * scheduler. + */ +static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q) +{ + struct blk_mq_hw_ctx *hctx; + + /* + * If the IO scheduler does not respect hardware queues when + * dispatching, we just don't bother with multiple HW queues and + * dispatch from hctx for the current CPU since running multiple queues + * just causes lock contention inside the scheduler and pointless cache + * bouncing. + */ + hctx = blk_mq_map_queue_type(q, HCTX_TYPE_DEFAULT, + raw_smp_processor_id()); + if (!blk_mq_hctx_stopped(hctx)) + return hctx; + return NULL; +} + /** * blk_mq_run_hw_queues - Run all hardware queues in a request queue. * @q: Pointer to the request queue to run. @@ -1653,14 +1689,23 @@ EXPORT_SYMBOL(blk_mq_run_hw_queue); */ void blk_mq_run_hw_queues(struct request_queue *q, bool async) { - struct blk_mq_hw_ctx *hctx; + struct blk_mq_hw_ctx *hctx, *sq_hctx; int i; + sq_hctx = NULL; + if (blk_mq_has_sqsched(q)) + sq_hctx = blk_mq_get_sq_hctx(q); queue_for_each_hw_ctx(q, hctx, i) { if (blk_mq_hctx_stopped(hctx)) continue; - - blk_mq_run_hw_queue(hctx, async); + /* + * Dispatch from this hctx either if there's no hctx preferred + * by IO scheduler or if it has requests that bypass the + * scheduler. + */ + if (!sq_hctx || sq_hctx == hctx || + !list_empty_careful(&hctx->dispatch)) + blk_mq_run_hw_queue(hctx, async); } } EXPORT_SYMBOL(blk_mq_run_hw_queues); @@ -1672,14 +1717,23 @@ EXPORT_SYMBOL(blk_mq_run_hw_queues); */ void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs) { - struct blk_mq_hw_ctx *hctx; + struct blk_mq_hw_ctx *hctx, *sq_hctx; int i; + sq_hctx = NULL; + if (blk_mq_has_sqsched(q)) + sq_hctx = blk_mq_get_sq_hctx(q); queue_for_each_hw_ctx(q, hctx, i) { if (blk_mq_hctx_stopped(hctx)) continue; - - blk_mq_delay_run_hw_queue(hctx, msecs); + /* + * Dispatch from this hctx either if there's no hctx preferred + * by IO scheduler or if it has requests that bypass the + * scheduler. + */ + if (!sq_hctx || sq_hctx == hctx || + !list_empty_careful(&hctx->dispatch)) + blk_mq_delay_run_hw_queue(hctx, msecs); } } EXPORT_SYMBOL(blk_mq_delay_run_hw_queues); diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index dc89199bc8c6..c25c41d0d061 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -1029,6 +1029,7 @@ static struct elevator_type kyber_sched = { #endif .elevator_attrs = kyber_sched_attrs, .elevator_name = "kyber", + .elevator_features = ELEVATOR_F_MQ_AWARE, .elevator_owner = THIS_MODULE, }; diff --git a/include/linux/elevator.h b/include/linux/elevator.h index bacc40a0bdf3..1fe8e105b83b 100644 --- a/include/linux/elevator.h +++ b/include/linux/elevator.h @@ -172,6 +172,8 @@ extern struct request *elv_rb_find(struct rb_root *, sector_t); /* Supports zoned block devices sequential write constraint */ #define ELEVATOR_F_ZBD_SEQ_WRITE (1U << 0) +/* Supports scheduling on multiple hardware queues */ +#define ELEVATOR_F_MQ_AWARE (1U << 1) #endif /* CONFIG_BLOCK */ #endif -- cgit v1.2.3 From 9f180e315a93cde559ac1c9c4c5ce980aa681c1c Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Mon, 11 Jan 2021 11:05:54 +0800 Subject: block: don't allocate inline bvecs if this bioset needn't bvecs The inline bvecs won't be used if user needn't bvecs by not passing BIOSET_NEED_BVECS, so don't allocate bvecs in this situation. Reviewed-by: Christoph Hellwig Signed-off-by: Ming Lei Reviewed-by: Pavel Begunkov Tested-by: Pavel Begunkov Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/bio.c | 7 +++++-- include/linux/bio.h | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/block/bio.c b/block/bio.c index 87bf16460e0e..8ccda51dd831 100644 --- a/block/bio.c +++ b/block/bio.c @@ -89,8 +89,7 @@ fail_alloc_slab: static inline unsigned int bs_bio_slab_size(struct bio_set *bs) { - return bs->front_pad + sizeof(struct bio) + - BIO_INLINE_VECS * sizeof(struct bio_vec); + return bs->front_pad + sizeof(struct bio) + bs->back_pad; } static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs) @@ -1563,6 +1562,10 @@ int bioset_init(struct bio_set *bs, int flags) { bs->front_pad = front_pad; + if (flags & BIOSET_NEED_BVECS) + bs->back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec); + else + bs->back_pad = 0; spin_lock_init(&bs->rescue_lock); bio_list_init(&bs->rescue_list); diff --git a/include/linux/bio.h b/include/linux/bio.h index 2f1155eabaff..5d8977aafa19 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -703,6 +703,7 @@ struct bio_set { mempool_t bvec_integrity_pool; #endif + unsigned int back_pad; /* * Deadlock avoidance for stacking block drivers: see comments in * bio_alloc_bioset() for details -- cgit v1.2.3 From eec716a1c18c796a69db0be5e2a6f282ba5bccd6 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Mon, 11 Jan 2021 11:05:56 +0800 Subject: block: move three bvec helpers declaration into private helper bvec_alloc(), bvec_free() and bvec_nr_vecs() are only used inside block layer core functions, no need to declare them in public header. Reviewed-by: Christoph Hellwig Signed-off-by: Ming Lei Reviewed-by: Pavel Begunkov Reviewed-by: Hannes Reinecke Signed-off-by: Jens Axboe --- block/blk.h | 4 ++++ include/linux/bio.h | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/block/blk.h b/block/blk.h index ab0aaf958553..0198335c5838 100644 --- a/block/blk.h +++ b/block/blk.h @@ -55,6 +55,10 @@ void blk_free_flush_queue(struct blk_flush_queue *q); void blk_freeze_queue(struct request_queue *q); +struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *); +void bvec_free(mempool_t *, struct bio_vec *, unsigned int); +unsigned int bvec_nr_vecs(unsigned short idx); + static inline bool biovec_phys_mergeable(struct request_queue *q, struct bio_vec *vec1, struct bio_vec *vec2) { diff --git a/include/linux/bio.h b/include/linux/bio.h index 5d8977aafa19..e135b500df5d 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -478,9 +478,6 @@ static inline void zero_fill_bio(struct bio *bio) zero_fill_bio_iter(bio, bio->bi_iter); } -extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *); -extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int); -extern unsigned int bvec_nr_vecs(unsigned short idx); extern const char *bio_devname(struct bio *bio, char *buffer); #define bio_set_dev(bio, bdev) \ -- cgit v1.2.3 From 8eeed0b554b9fda61be05b17cbb0b89ea2cbbb65 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Mon, 25 Jan 2021 05:49:57 +0100 Subject: block: remove unnecessary argument from blk_execute_rq_nowait The 'q' is not used since commit a1ce35fa4985 ("block: remove dead elevator code"), also update the comment of the function. And more importantly it never really was needed to start with given that we can trivial derive it from struct request. Cc: target-devel@vger.kernel.org Cc: linux-scsi@vger.kernel.org Cc: virtualization@lists.linux-foundation.org Cc: linux-ide@vger.kernel.org Cc: linux-mmc@vger.kernel.org Cc: linux-nvme@lists.infradead.org Cc: linux-nfs@vger.kernel.org Signed-off-by: Guoqing Jiang Signed-off-by: Jens Axboe --- block/blk-exec.c | 10 ++++------ drivers/block/sx8.c | 4 ++-- drivers/nvme/host/core.c | 4 ++-- drivers/nvme/host/lightnvm.c | 2 +- drivers/nvme/host/pci.c | 4 ++-- drivers/nvme/target/passthru.c | 2 +- drivers/scsi/scsi_error.c | 2 +- drivers/scsi/sg.c | 3 +-- drivers/scsi/st.c | 2 +- drivers/target/target_core_pscsi.c | 3 +-- include/linux/blkdev.h | 2 +- 11 files changed, 17 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/block/blk-exec.c b/block/blk-exec.c index 85324d53d072..2e37e85456fb 100644 --- a/block/blk-exec.c +++ b/block/blk-exec.c @@ -31,8 +31,7 @@ static void blk_end_sync_rq(struct request *rq, blk_status_t error) } /** - * blk_execute_rq_nowait - insert a request into queue for execution - * @q: queue to insert the request in + * blk_execute_rq_nowait - insert a request to I/O scheduler for execution * @bd_disk: matching gendisk * @rq: request to insert * @at_head: insert request at head or tail of queue @@ -45,9 +44,8 @@ static void blk_end_sync_rq(struct request *rq, blk_status_t error) * Note: * This function will invoke @done directly if the queue is dead. */ -void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk, - struct request *rq, int at_head, - rq_end_io_fn *done) +void blk_execute_rq_nowait(struct gendisk *bd_disk, struct request *rq, + int at_head, rq_end_io_fn *done) { WARN_ON(irqs_disabled()); WARN_ON(!blk_rq_is_passthrough(rq)); @@ -83,7 +81,7 @@ void blk_execute_rq(struct request_queue *q, struct gendisk *bd_disk, unsigned long hang_check; rq->end_io_data = &wait; - blk_execute_rq_nowait(q, bd_disk, rq, at_head, blk_end_sync_rq); + blk_execute_rq_nowait(bd_disk, rq, at_head, blk_end_sync_rq); /* Prevent hang_check timer from firing at us during very long I/O */ hang_check = sysctl_hung_task_timeout_secs; diff --git a/drivers/block/sx8.c b/drivers/block/sx8.c index 4478eb7efee0..2cdf2771f8e8 100644 --- a/drivers/block/sx8.c +++ b/drivers/block/sx8.c @@ -539,7 +539,7 @@ static int carm_array_info (struct carm_host *host, unsigned int array_idx) spin_unlock_irq(&host->lock); DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag); - blk_execute_rq_nowait(host->oob_q, NULL, rq, true, NULL); + blk_execute_rq_nowait(NULL, rq, true, NULL); return 0; @@ -578,7 +578,7 @@ static int carm_send_special (struct carm_host *host, carm_sspc_t func) crq->msg_bucket = (u32) rc; DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag); - blk_execute_rq_nowait(host->oob_q, NULL, rq, true, NULL); + blk_execute_rq_nowait(NULL, rq, true, NULL); return 0; } diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index a39befb4deba..0bea9ae03092 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -925,7 +925,7 @@ static void nvme_execute_rq_polled(struct request_queue *q, rq->cmd_flags |= REQ_HIPRI; rq->end_io_data = &wait; - blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq); + blk_execute_rq_nowait(bd_disk, rq, at_head, nvme_end_sync_rq); while (!completion_done(&wait)) { blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true); @@ -1202,7 +1202,7 @@ static int nvme_keep_alive(struct nvme_ctrl *ctrl) rq->timeout = ctrl->kato * HZ; rq->end_io_data = ctrl; - blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io); + blk_execute_rq_nowait(NULL, rq, 0, nvme_keep_alive_end_io); return 0; } diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c index 6c8eab8de288..0e5a55075e35 100644 --- a/drivers/nvme/host/lightnvm.c +++ b/drivers/nvme/host/lightnvm.c @@ -695,7 +695,7 @@ static int nvme_nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd, rq->end_io_data = rqd; - blk_execute_rq_nowait(q, NULL, rq, 0, nvme_nvm_end_io); + blk_execute_rq_nowait(NULL, rq, 0, nvme_nvm_end_io); return 0; diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 856aa31931c1..5b78e68be9a1 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1357,7 +1357,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved) } abort_req->end_io_data = NULL; - blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio); + blk_execute_rq_nowait(NULL, abort_req, 0, abort_endio); /* * The aborted req will be completed on receiving the abort req. @@ -2281,7 +2281,7 @@ static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode) req->end_io_data = nvmeq; init_completion(&nvmeq->delete_done); - blk_execute_rq_nowait(q, NULL, req, false, + blk_execute_rq_nowait(NULL, req, false, opcode == nvme_admin_delete_cq ? nvme_del_cq_end : nvme_del_queue_end); return 0; diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c index b9776fc8f08f..cbc88acdd233 100644 --- a/drivers/nvme/target/passthru.c +++ b/drivers/nvme/target/passthru.c @@ -275,7 +275,7 @@ static void nvmet_passthru_execute_cmd(struct nvmet_req *req) schedule_work(&req->p.work); } else { rq->end_io_data = req; - blk_execute_rq_nowait(rq->q, ns ? ns->disk : NULL, rq, 0, + blk_execute_rq_nowait(ns ? ns->disk : NULL, rq, 0, nvmet_passthru_req_done); } diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c index f11f51e2465f..c00f06e9ecb0 100644 --- a/drivers/scsi/scsi_error.c +++ b/drivers/scsi/scsi_error.c @@ -2007,7 +2007,7 @@ static void scsi_eh_lock_door(struct scsi_device *sdev) req->timeout = 10 * HZ; rq->retries = 5; - blk_execute_rq_nowait(req->q, NULL, req, 1, eh_lock_door_done); + blk_execute_rq_nowait(NULL, req, 1, eh_lock_door_done); } /** diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index bfa8d77322d7..4383d93110f8 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -829,8 +829,7 @@ sg_common_write(Sg_fd * sfp, Sg_request * srp, srp->rq->timeout = timeout; kref_get(&sfp->f_ref); /* sg_rq_end_io() does kref_put(). */ - blk_execute_rq_nowait(sdp->device->request_queue, sdp->disk, - srp->rq, at_head, sg_rq_end_io); + blk_execute_rq_nowait(sdp->disk, srp->rq, at_head, sg_rq_end_io); return 0; } diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c index 43f7624508a9..841ad2fc369a 100644 --- a/drivers/scsi/st.c +++ b/drivers/scsi/st.c @@ -585,7 +585,7 @@ static int st_scsi_execute(struct st_request *SRpnt, const unsigned char *cmd, rq->retries = retries; req->end_io_data = SRpnt; - blk_execute_rq_nowait(req->q, NULL, req, 1, st_scsi_execute_end); + blk_execute_rq_nowait(NULL, req, 1, st_scsi_execute_end); return 0; } diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c index 7994f27e4527..33770e5808ce 100644 --- a/drivers/target/target_core_pscsi.c +++ b/drivers/target/target_core_pscsi.c @@ -1000,8 +1000,7 @@ pscsi_execute_cmd(struct se_cmd *cmd) req->timeout = PS_TIMEOUT_OTHER; scsi_req(req)->retries = PS_RETRY; - blk_execute_rq_nowait(pdv->pdv_sd->request_queue, NULL, req, - (cmd->sam_task_attr == TCM_HEAD_TAG), + blk_execute_rq_nowait(NULL, req, (cmd->sam_task_attr == TCM_HEAD_TAG), pscsi_req_done); return 0; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 4526b9ef8edb..623a61239429 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -950,7 +950,7 @@ extern int blk_rq_map_user_iov(struct request_queue *, struct request *, gfp_t); extern void blk_execute_rq(struct request_queue *, struct gendisk *, struct request *, int); -extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *, +extern void blk_execute_rq_nowait(struct gendisk *, struct request *, int, rq_end_io_fn *); /* Helper to convert REQ_OP_XXX to its string format XXX */ -- cgit v1.2.3 From 684da7628d93bbdcfba9081b917d99f29ad04c23 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Mon, 25 Jan 2021 05:49:58 +0100 Subject: block: remove unnecessary argument from blk_execute_rq We can remove 'q' from blk_execute_rq as well after the previous change in blk_execute_rq_nowait. And more importantly it never really was needed to start with given that we can trivial derive it from struct request. Cc: linux-scsi@vger.kernel.org Cc: virtualization@lists.linux-foundation.org Cc: linux-ide@vger.kernel.org Cc: linux-mmc@vger.kernel.org Cc: linux-nvme@lists.infradead.org Cc: linux-nfs@vger.kernel.org Acked-by: Ulf Hansson # for mmc Signed-off-by: Guoqing Jiang Signed-off-by: Jens Axboe --- block/blk-exec.c | 3 +-- block/bsg.c | 2 +- block/scsi_ioctl.c | 6 +++--- drivers/block/mtip32xx/mtip32xx.c | 2 +- drivers/block/paride/pd.c | 2 +- drivers/block/pktcdvd.c | 2 +- drivers/block/virtio_blk.c | 2 +- drivers/cdrom/cdrom.c | 2 +- drivers/ide/ide-atapi.c | 2 +- drivers/ide/ide-cd.c | 2 +- drivers/ide/ide-cd_ioctl.c | 2 +- drivers/ide/ide-devsets.c | 2 +- drivers/ide/ide-disk.c | 2 +- drivers/ide/ide-ioctls.c | 4 ++-- drivers/ide/ide-park.c | 2 +- drivers/ide/ide-pm.c | 4 ++-- drivers/ide/ide-tape.c | 2 +- drivers/ide/ide-taskfile.c | 2 +- drivers/mmc/core/block.c | 10 +++++----- drivers/nvme/host/core.c | 4 ++-- drivers/nvme/host/lightnvm.c | 2 +- drivers/scsi/scsi_lib.c | 2 +- fs/nfsd/blocklayout.c | 2 +- include/linux/blkdev.h | 3 +-- 24 files changed, 33 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/block/blk-exec.c b/block/blk-exec.c index 2e37e85456fb..0ab873f10133 100644 --- a/block/blk-exec.c +++ b/block/blk-exec.c @@ -74,8 +74,7 @@ EXPORT_SYMBOL_GPL(blk_execute_rq_nowait); * Insert a fully prepared request at the back of the I/O scheduler queue * for execution and wait for completion. */ -void blk_execute_rq(struct request_queue *q, struct gendisk *bd_disk, - struct request *rq, int at_head) +void blk_execute_rq(struct gendisk *bd_disk, struct request *rq, int at_head) { DECLARE_COMPLETION_ONSTACK(wait); unsigned long hang_check; diff --git a/block/bsg.c b/block/bsg.c index 3d78e843a83f..bd10922d5cbb 100644 --- a/block/bsg.c +++ b/block/bsg.c @@ -183,7 +183,7 @@ static int bsg_sg_io(struct request_queue *q, fmode_t mode, void __user *uarg) bio = rq->bio; - blk_execute_rq(q, NULL, rq, !(hdr.flags & BSG_FLAG_Q_AT_TAIL)); + blk_execute_rq(NULL, rq, !(hdr.flags & BSG_FLAG_Q_AT_TAIL)); ret = rq->q->bsg_dev.ops->complete_rq(rq, &hdr); blk_rq_unmap_user(bio); diff --git a/block/scsi_ioctl.c b/block/scsi_ioctl.c index c9f009cc0446..6599bac0a78c 100644 --- a/block/scsi_ioctl.c +++ b/block/scsi_ioctl.c @@ -357,7 +357,7 @@ static int sg_io(struct request_queue *q, struct gendisk *bd_disk, * (if he doesn't check that is his problem). * N.B. a non-zero SCSI status is _not_ necessarily an error. */ - blk_execute_rq(q, bd_disk, rq, at_head); + blk_execute_rq(bd_disk, rq, at_head); hdr->duration = jiffies_to_msecs(jiffies - start_time); @@ -493,7 +493,7 @@ int sg_scsi_ioctl(struct request_queue *q, struct gendisk *disk, fmode_t mode, goto error; } - blk_execute_rq(q, disk, rq, 0); + blk_execute_rq(disk, rq, 0); err = req->result & 0xff; /* only 8 bit SCSI status */ if (err) { @@ -532,7 +532,7 @@ static int __blk_send_generic(struct request_queue *q, struct gendisk *bd_disk, scsi_req(rq)->cmd[0] = cmd; scsi_req(rq)->cmd[4] = data; scsi_req(rq)->cmd_len = 6; - blk_execute_rq(q, bd_disk, rq, 0); + blk_execute_rq(bd_disk, rq, 0); err = scsi_req(rq)->result ? -EIO : 0; blk_put_request(rq); diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 53ac59d19ae5..3fd99836bb1c 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -1015,7 +1015,7 @@ static int mtip_exec_internal_command(struct mtip_port *port, rq->timeout = timeout; /* insert request and run queue */ - blk_execute_rq(rq->q, NULL, rq, true); + blk_execute_rq(NULL, rq, true); if (int_cmd->status) { dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n", diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c index a7af4f27b7c3..897acda20ac8 100644 --- a/drivers/block/paride/pd.c +++ b/drivers/block/paride/pd.c @@ -781,7 +781,7 @@ static int pd_special_command(struct pd_unit *disk, req = blk_mq_rq_to_pdu(rq); req->func = func; - blk_execute_rq(disk->gd->queue, disk->gd, rq, 0); + blk_execute_rq(disk->gd, rq, 0); blk_put_request(rq); return 0; } diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 658a0981cb54..fc4b0f1aa86d 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -722,7 +722,7 @@ static int pkt_generic_packet(struct pktcdvd_device *pd, struct packet_command * if (cgc->quiet) rq->rq_flags |= RQF_QUIET; - blk_execute_rq(rq->q, pd->bdev->bd_disk, rq, 0); + blk_execute_rq(pd->bdev->bd_disk, rq, 0); if (scsi_req(rq)->result) ret = -EIO; out: diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 145606dc52db..b0285db7cf4f 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -320,7 +320,7 @@ static int virtblk_get_id(struct gendisk *disk, char *id_str) if (err) goto out; - blk_execute_rq(vblk->disk->queue, vblk->disk, req, false); + blk_execute_rq(vblk->disk, req, false); err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req))); out: blk_put_request(req); diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c index 8f0e52a71493..90ad34c6ef8e 100644 --- a/drivers/cdrom/cdrom.c +++ b/drivers/cdrom/cdrom.c @@ -2214,7 +2214,7 @@ static int cdrom_read_cdda_bpc(struct cdrom_device_info *cdi, __u8 __user *ubuf, rq->timeout = 60 * HZ; bio = rq->bio; - blk_execute_rq(q, cdi->disk, rq, 0); + blk_execute_rq(cdi->disk, rq, 0); if (scsi_req(rq)->result) { struct scsi_sense_hdr sshdr; diff --git a/drivers/ide/ide-atapi.c b/drivers/ide/ide-atapi.c index 013ad33fbbc8..a1ce9f5ac3aa 100644 --- a/drivers/ide/ide-atapi.c +++ b/drivers/ide/ide-atapi.c @@ -107,7 +107,7 @@ int ide_queue_pc_tail(ide_drive_t *drive, struct gendisk *disk, memcpy(scsi_req(rq)->cmd, pc->c, 12); if (drive->media == ide_tape) scsi_req(rq)->cmd[13] = REQ_IDETAPE_PC1; - blk_execute_rq(drive->queue, disk, rq, 0); + blk_execute_rq(disk, rq, 0); error = scsi_req(rq)->result ? -EIO : 0; put_req: blk_put_request(rq); diff --git a/drivers/ide/ide-cd.c b/drivers/ide/ide-cd.c index 25d2d88e82ad..cffbcc27a34c 100644 --- a/drivers/ide/ide-cd.c +++ b/drivers/ide/ide-cd.c @@ -467,7 +467,7 @@ int ide_cd_queue_pc(ide_drive_t *drive, const unsigned char *cmd, } } - blk_execute_rq(drive->queue, info->disk, rq, 0); + blk_execute_rq(info->disk, rq, 0); error = scsi_req(rq)->result ? -EIO : 0; if (buffer) diff --git a/drivers/ide/ide-cd_ioctl.c b/drivers/ide/ide-cd_ioctl.c index 46f2df288c6a..011eab9c69b7 100644 --- a/drivers/ide/ide-cd_ioctl.c +++ b/drivers/ide/ide-cd_ioctl.c @@ -299,7 +299,7 @@ int ide_cdrom_reset(struct cdrom_device_info *cdi) rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0); ide_req(rq)->type = ATA_PRIV_MISC; rq->rq_flags = RQF_QUIET; - blk_execute_rq(drive->queue, cd->disk, rq, 0); + blk_execute_rq(cd->disk, rq, 0); ret = scsi_req(rq)->result ? -EIO : 0; blk_put_request(rq); /* diff --git a/drivers/ide/ide-devsets.c b/drivers/ide/ide-devsets.c index f2f93ed40356..ca1d4b3d3878 100644 --- a/drivers/ide/ide-devsets.c +++ b/drivers/ide/ide-devsets.c @@ -173,7 +173,7 @@ int ide_devset_execute(ide_drive_t *drive, const struct ide_devset *setting, *(int *)&scsi_req(rq)->cmd[1] = arg; ide_req(rq)->special = setting->set; - blk_execute_rq(q, NULL, rq, 0); + blk_execute_rq(NULL, rq, 0); ret = scsi_req(rq)->result; blk_put_request(rq); diff --git a/drivers/ide/ide-disk.c b/drivers/ide/ide-disk.c index 34b9441084f8..8413731c6259 100644 --- a/drivers/ide/ide-disk.c +++ b/drivers/ide/ide-disk.c @@ -482,7 +482,7 @@ static int set_multcount(ide_drive_t *drive, int arg) drive->mult_req = arg; drive->special_flags |= IDE_SFLAG_SET_MULTMODE; - blk_execute_rq(drive->queue, NULL, rq, 0); + blk_execute_rq(NULL, rq, 0); blk_put_request(rq); return (drive->mult_count == arg) ? 0 : -EIO; diff --git a/drivers/ide/ide-ioctls.c b/drivers/ide/ide-ioctls.c index 58994da10c06..43fbc37d85c3 100644 --- a/drivers/ide/ide-ioctls.c +++ b/drivers/ide/ide-ioctls.c @@ -137,7 +137,7 @@ static int ide_cmd_ioctl(ide_drive_t *drive, void __user *argp) rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0); ide_req(rq)->type = ATA_PRIV_TASKFILE; - blk_execute_rq(drive->queue, NULL, rq, 0); + blk_execute_rq(NULL, rq, 0); err = scsi_req(rq)->result ? -EIO : 0; blk_put_request(rq); @@ -235,7 +235,7 @@ static int generic_drive_reset(ide_drive_t *drive) ide_req(rq)->type = ATA_PRIV_MISC; scsi_req(rq)->cmd_len = 1; scsi_req(rq)->cmd[0] = REQ_DRIVE_RESET; - blk_execute_rq(drive->queue, NULL, rq, 1); + blk_execute_rq(NULL, rq, 1); ret = scsi_req(rq)->result; blk_put_request(rq); return ret; diff --git a/drivers/ide/ide-park.c b/drivers/ide/ide-park.c index 8af7af6001eb..a80a0f28f7b9 100644 --- a/drivers/ide/ide-park.c +++ b/drivers/ide/ide-park.c @@ -37,7 +37,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout) scsi_req(rq)->cmd_len = 1; ide_req(rq)->type = ATA_PRIV_MISC; ide_req(rq)->special = &timeout; - blk_execute_rq(q, NULL, rq, 1); + blk_execute_rq(NULL, rq, 1); rc = scsi_req(rq)->result ? -EIO : 0; blk_put_request(rq); if (rc) diff --git a/drivers/ide/ide-pm.c b/drivers/ide/ide-pm.c index 82ab308f1aaf..d680b3e3295f 100644 --- a/drivers/ide/ide-pm.c +++ b/drivers/ide/ide-pm.c @@ -27,7 +27,7 @@ int generic_ide_suspend(struct device *dev, pm_message_t mesg) mesg.event = PM_EVENT_FREEZE; rqpm.pm_state = mesg.event; - blk_execute_rq(drive->queue, NULL, rq, 0); + blk_execute_rq(NULL, rq, 0); ret = scsi_req(rq)->result ? -EIO : 0; blk_put_request(rq); @@ -50,7 +50,7 @@ static int ide_pm_execute_rq(struct request *rq) blk_mq_end_request(rq, BLK_STS_OK); return -ENXIO; } - blk_execute_rq(q, NULL, rq, true); + blk_execute_rq(NULL, rq, true); return scsi_req(rq)->result ? -EIO : 0; } diff --git a/drivers/ide/ide-tape.c b/drivers/ide/ide-tape.c index 88b96437b22e..fa05e7e7d609 100644 --- a/drivers/ide/ide-tape.c +++ b/drivers/ide/ide-tape.c @@ -868,7 +868,7 @@ static int idetape_queue_rw_tail(ide_drive_t *drive, int cmd, int size) goto out_put; } - blk_execute_rq(drive->queue, tape->disk, rq, 0); + blk_execute_rq(tape->disk, rq, 0); /* calculate the number of transferred bytes and update buffer state */ size -= scsi_req(rq)->resid_len; diff --git a/drivers/ide/ide-taskfile.c b/drivers/ide/ide-taskfile.c index d016cbe68cba..6665fc4724b9 100644 --- a/drivers/ide/ide-taskfile.c +++ b/drivers/ide/ide-taskfile.c @@ -443,7 +443,7 @@ int ide_raw_taskfile(ide_drive_t *drive, struct ide_cmd *cmd, u8 *buf, ide_req(rq)->special = cmd; cmd->rq = rq; - blk_execute_rq(drive->queue, NULL, rq, 0); + blk_execute_rq(NULL, rq, 0); error = scsi_req(rq)->result ? -EIO : 0; put_req: blk_put_request(rq); diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 42e27a298218..a1d6b68320ae 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -253,7 +253,7 @@ static ssize_t power_ro_lock_store(struct device *dev, goto out_put; } req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_BOOT_WP; - blk_execute_rq(mq->queue, NULL, req, 0); + blk_execute_rq(NULL, req, 0); ret = req_to_mmc_queue_req(req)->drv_op_result; blk_put_request(req); @@ -629,7 +629,7 @@ static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md, rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL; req_to_mmc_queue_req(req)->drv_op_data = idatas; req_to_mmc_queue_req(req)->ioc_count = 1; - blk_execute_rq(mq->queue, NULL, req, 0); + blk_execute_rq(NULL, req, 0); ioc_err = req_to_mmc_queue_req(req)->drv_op_result; err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata); blk_put_request(req); @@ -698,7 +698,7 @@ static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md, rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL; req_to_mmc_queue_req(req)->drv_op_data = idata; req_to_mmc_queue_req(req)->ioc_count = num_of_cmds; - blk_execute_rq(mq->queue, NULL, req, 0); + blk_execute_rq(NULL, req, 0); ioc_err = req_to_mmc_queue_req(req)->drv_op_result; /* copy to user if data and response */ @@ -2722,7 +2722,7 @@ static int mmc_dbg_card_status_get(void *data, u64 *val) if (IS_ERR(req)) return PTR_ERR(req); req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS; - blk_execute_rq(mq->queue, NULL, req, 0); + blk_execute_rq(NULL, req, 0); ret = req_to_mmc_queue_req(req)->drv_op_result; if (ret >= 0) { *val = ret; @@ -2761,7 +2761,7 @@ static int mmc_ext_csd_open(struct inode *inode, struct file *filp) } req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD; req_to_mmc_queue_req(req)->drv_op_data = &ext_csd; - blk_execute_rq(mq->queue, NULL, req, 0); + blk_execute_rq(NULL, req, 0); err = req_to_mmc_queue_req(req)->drv_op_result; blk_put_request(req); if (err) { diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 0bea9ae03092..eb7963fb167b 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -964,7 +964,7 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, if (poll) nvme_execute_rq_polled(req->q, NULL, req, at_head); else - blk_execute_rq(req->q, NULL, req, at_head); + blk_execute_rq(NULL, req, at_head); if (result) *result = nvme_req(req)->result; if (nvme_req(req)->flags & NVME_REQ_CANCELLED) @@ -1101,7 +1101,7 @@ void nvme_execute_passthru_rq(struct request *rq) u32 effects; effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode); - blk_execute_rq(rq->q, disk, rq, 0); + blk_execute_rq(disk, rq, 0); nvme_passthru_end(ctrl, effects); } EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU); diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c index 0e5a55075e35..ec38128f51e9 100644 --- a/drivers/nvme/host/lightnvm.c +++ b/drivers/nvme/host/lightnvm.c @@ -819,7 +819,7 @@ static int nvme_nvm_submit_user_cmd(struct request_queue *q, bio->bi_bdev = ns->disk->part0; } - blk_execute_rq(q, NULL, rq, 0); + blk_execute_rq(NULL, rq, 0); if (nvme_req(rq)->flags & NVME_REQ_CANCELLED) ret = -EINTR; diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index b3f14f05340a..4d2280658559 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -269,7 +269,7 @@ int __scsi_execute(struct scsi_device *sdev, const unsigned char *cmd, /* * head injection *required* here otherwise quiesce won't work */ - blk_execute_rq(req->q, NULL, req, 1); + blk_execute_rq(NULL, req, 1); /* * Some devices (USB mass-storage in particular) may transfer diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c index a07c39c94bbd..1058659a8d31 100644 --- a/fs/nfsd/blocklayout.c +++ b/fs/nfsd/blocklayout.c @@ -254,7 +254,7 @@ again: req->cmd[4] = bufflen & 0xff; req->cmd_len = COMMAND_SIZE(INQUIRY); - blk_execute_rq(rq->q, NULL, rq, 1); + blk_execute_rq(NULL, rq, 1); if (req->result) { pr_err("pNFS: INQUIRY 0x83 failed with: %x\n", req->result); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 623a61239429..20f3706b6b2e 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -948,8 +948,7 @@ extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, uns extern int blk_rq_map_user_iov(struct request_queue *, struct request *, struct rq_map_data *, const struct iov_iter *, gfp_t); -extern void blk_execute_rq(struct request_queue *, struct gendisk *, - struct request *, int); +extern void blk_execute_rq(struct gendisk *, struct request *, int); extern void blk_execute_rq_nowait(struct gendisk *, struct request *, int, rq_end_io_fn *); -- cgit v1.2.3 From f75a1025c0b94d58de56c9421104ed6e987ddcd2 Mon Sep 17 00:00:00 2001 From: Kyle Tso Date: Sat, 16 Jan 2021 00:33:11 +0800 Subject: usb: typec: tcpm: Create legacy PDOs for PD2 connection If the port partner is PD2, the PDOs of the local port should follow the format defined in PD2 Spec. Dynamically modify the pre-defined PD3 PDOs and transform them into PD2 format before sending them to the PD2 port partner. Reviewed-by: Guenter Roeck Signed-off-by: Kyle Tso Link: https://lore.kernel.org/r/20210115163311.391332-1-kyletso@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tcpm/tcpm.c | 62 ++++++++++++++++++++++++++++++++++++------- include/linux/usb/pd.h | 1 + 2 files changed, 53 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 0dd932fe08d0..0afd8ef692e8 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -1023,13 +1023,47 @@ static int tcpm_set_pwr_role(struct tcpm_port *port, enum typec_role role) return 0; } +/* + * Transform the PDO to be compliant to PD rev2.0. + * Return 0 if the PDO type is not defined in PD rev2.0. + * Otherwise, return the converted PDO. + */ +static u32 tcpm_forge_legacy_pdo(struct tcpm_port *port, u32 pdo, enum typec_role role) +{ + switch (pdo_type(pdo)) { + case PDO_TYPE_FIXED: + if (role == TYPEC_SINK) + return pdo & ~PDO_FIXED_FRS_CURR_MASK; + else + return pdo & ~PDO_FIXED_UNCHUNK_EXT; + case PDO_TYPE_VAR: + case PDO_TYPE_BATT: + return pdo; + case PDO_TYPE_APDO: + default: + return 0; + } +} + static int tcpm_pd_send_source_caps(struct tcpm_port *port) { struct pd_message msg; - int i; + u32 pdo; + unsigned int i, nr_pdo = 0; memset(&msg, 0, sizeof(msg)); - if (!port->nr_src_pdo) { + + for (i = 0; i < port->nr_src_pdo; i++) { + if (port->negotiated_rev >= PD_REV30) { + msg.payload[nr_pdo++] = cpu_to_le32(port->src_pdo[i]); + } else { + pdo = tcpm_forge_legacy_pdo(port, port->src_pdo[i], TYPEC_SOURCE); + if (pdo) + msg.payload[nr_pdo++] = cpu_to_le32(pdo); + } + } + + if (!nr_pdo) { /* No source capabilities defined, sink only */ msg.header = PD_HEADER_LE(PD_CTRL_REJECT, port->pwr_role, @@ -1042,10 +1076,8 @@ static int tcpm_pd_send_source_caps(struct tcpm_port *port) port->data_role, port->negotiated_rev, port->message_id, - port->nr_src_pdo); + nr_pdo); } - for (i = 0; i < port->nr_src_pdo; i++) - msg.payload[i] = cpu_to_le32(port->src_pdo[i]); return tcpm_pd_transmit(port, TCPC_TX_SOP, &msg); } @@ -1053,10 +1085,22 @@ static int tcpm_pd_send_source_caps(struct tcpm_port *port) static int tcpm_pd_send_sink_caps(struct tcpm_port *port) { struct pd_message msg; - int i; + u32 pdo; + unsigned int i, nr_pdo = 0; memset(&msg, 0, sizeof(msg)); - if (!port->nr_snk_pdo) { + + for (i = 0; i < port->nr_snk_pdo; i++) { + if (port->negotiated_rev >= PD_REV30) { + msg.payload[nr_pdo++] = cpu_to_le32(port->snk_pdo[i]); + } else { + pdo = tcpm_forge_legacy_pdo(port, port->snk_pdo[i], TYPEC_SINK); + if (pdo) + msg.payload[nr_pdo++] = cpu_to_le32(pdo); + } + } + + if (!nr_pdo) { /* No sink capabilities defined, source only */ msg.header = PD_HEADER_LE(PD_CTRL_REJECT, port->pwr_role, @@ -1069,10 +1113,8 @@ static int tcpm_pd_send_sink_caps(struct tcpm_port *port) port->data_role, port->negotiated_rev, port->message_id, - port->nr_snk_pdo); + nr_pdo); } - for (i = 0; i < port->nr_snk_pdo; i++) - msg.payload[i] = cpu_to_le32(port->snk_pdo[i]); return tcpm_pd_transmit(port, TCPC_TX_SOP, &msg); } diff --git a/include/linux/usb/pd.h b/include/linux/usb/pd.h index 272454f9cd67..70d681918d01 100644 --- a/include/linux/usb/pd.h +++ b/include/linux/usb/pd.h @@ -225,6 +225,7 @@ enum pd_pdo_type { #define PDO_FIXED_EXTPOWER BIT(27) /* Externally powered */ #define PDO_FIXED_USB_COMM BIT(26) /* USB communications capable */ #define PDO_FIXED_DATA_SWAP BIT(25) /* Data role swap supported */ +#define PDO_FIXED_UNCHUNK_EXT BIT(24) /* Unchunked Extended Message supported (Source) */ #define PDO_FIXED_FRS_CURR_MASK (BIT(24) | BIT(23)) /* FR_Swap Current (Sink) */ #define PDO_FIXED_FRS_CURR_SHIFT 23 #define PDO_FIXED_VOLT_SHIFT 10 /* 50mV units */ -- cgit v1.2.3 From 2289e87b5951f97783f07fc895e6c5e804b53668 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Thu, 17 Sep 2020 17:22:49 -0400 Subject: SUNRPC: Make trace_svc_process() display the RPC procedure symbolically The next few patches will employ these strings to help make server- side trace logs more human-readable. A similar technique is already in use in kernel RPC client code. Signed-off-by: Chuck Lever --- fs/lockd/svc4proc.c | 24 ++++++++++++++++++++++++ fs/lockd/svcproc.c | 24 ++++++++++++++++++++++++ fs/nfs/callback_xdr.c | 2 ++ fs/nfsd/nfs2acl.c | 5 +++++ fs/nfsd/nfs3acl.c | 3 +++ fs/nfsd/nfs3proc.c | 22 ++++++++++++++++++++++ fs/nfsd/nfs4proc.c | 2 ++ fs/nfsd/nfsproc.c | 18 ++++++++++++++++++ include/linux/sunrpc/svc.h | 1 + 9 files changed, 101 insertions(+) (limited to 'include/linux') diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c index fa41dda39925..4c10fb5138f1 100644 --- a/fs/lockd/svc4proc.c +++ b/fs/lockd/svc4proc.c @@ -512,6 +512,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "NULL", }, [NLMPROC_TEST] = { .pc_func = nlm4svc_proc_test, @@ -520,6 +521,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+2+No+Rg, + .pc_name = "TEST", }, [NLMPROC_LOCK] = { .pc_func = nlm4svc_proc_lock, @@ -528,6 +530,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "LOCK", }, [NLMPROC_CANCEL] = { .pc_func = nlm4svc_proc_cancel, @@ -536,6 +539,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "CANCEL", }, [NLMPROC_UNLOCK] = { .pc_func = nlm4svc_proc_unlock, @@ -544,6 +548,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "UNLOCK", }, [NLMPROC_GRANTED] = { .pc_func = nlm4svc_proc_granted, @@ -552,6 +557,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "GRANTED", }, [NLMPROC_TEST_MSG] = { .pc_func = nlm4svc_proc_test_msg, @@ -560,6 +566,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "TEST_MSG", }, [NLMPROC_LOCK_MSG] = { .pc_func = nlm4svc_proc_lock_msg, @@ -568,6 +575,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "LOCK_MSG", }, [NLMPROC_CANCEL_MSG] = { .pc_func = nlm4svc_proc_cancel_msg, @@ -576,6 +584,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "CANCEL_MSG", }, [NLMPROC_UNLOCK_MSG] = { .pc_func = nlm4svc_proc_unlock_msg, @@ -584,6 +593,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNLOCK_MSG", }, [NLMPROC_GRANTED_MSG] = { .pc_func = nlm4svc_proc_granted_msg, @@ -592,6 +602,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "GRANTED_MSG", }, [NLMPROC_TEST_RES] = { .pc_func = nlm4svc_proc_null, @@ -600,6 +611,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "TEST_RES", }, [NLMPROC_LOCK_RES] = { .pc_func = nlm4svc_proc_null, @@ -608,6 +620,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "LOCK_RES", }, [NLMPROC_CANCEL_RES] = { .pc_func = nlm4svc_proc_null, @@ -616,6 +629,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "CANCEL_RES", }, [NLMPROC_UNLOCK_RES] = { .pc_func = nlm4svc_proc_null, @@ -624,6 +638,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNLOCK_RES", }, [NLMPROC_GRANTED_RES] = { .pc_func = nlm4svc_proc_granted_res, @@ -632,6 +647,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "GRANTED_RES", }, [NLMPROC_NSM_NOTIFY] = { .pc_func = nlm4svc_proc_sm_notify, @@ -640,6 +656,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_reboot), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "SM_NOTIFY", }, [17] = { .pc_func = nlm4svc_proc_unused, @@ -648,6 +665,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = 0, + .pc_name = "UNUSED", }, [18] = { .pc_func = nlm4svc_proc_unused, @@ -656,6 +674,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = 0, + .pc_name = "UNUSED", }, [19] = { .pc_func = nlm4svc_proc_unused, @@ -664,6 +683,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = 0, + .pc_name = "UNUSED", }, [NLMPROC_SHARE] = { .pc_func = nlm4svc_proc_share, @@ -672,6 +692,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+1, + .pc_name = "SHARE", }, [NLMPROC_UNSHARE] = { .pc_func = nlm4svc_proc_unshare, @@ -680,6 +701,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+1, + .pc_name = "UNSHARE", }, [NLMPROC_NM_LOCK] = { .pc_func = nlm4svc_proc_nm_lock, @@ -688,6 +710,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "NM_LOCK", }, [NLMPROC_FREE_ALL] = { .pc_func = nlm4svc_proc_free_all, @@ -696,5 +719,6 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "FREE_ALL", }, }; diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c index 50855f2c1f4b..4ae4b63b5392 100644 --- a/fs/lockd/svcproc.c +++ b/fs/lockd/svcproc.c @@ -554,6 +554,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "NULL", }, [NLMPROC_TEST] = { .pc_func = nlmsvc_proc_test, @@ -562,6 +563,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+2+No+Rg, + .pc_name = "TEST", }, [NLMPROC_LOCK] = { .pc_func = nlmsvc_proc_lock, @@ -570,6 +572,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "LOCK", }, [NLMPROC_CANCEL] = { .pc_func = nlmsvc_proc_cancel, @@ -578,6 +581,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "CANCEL", }, [NLMPROC_UNLOCK] = { .pc_func = nlmsvc_proc_unlock, @@ -586,6 +590,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "UNLOCK", }, [NLMPROC_GRANTED] = { .pc_func = nlmsvc_proc_granted, @@ -594,6 +599,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "GRANTED", }, [NLMPROC_TEST_MSG] = { .pc_func = nlmsvc_proc_test_msg, @@ -602,6 +608,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "TEST_MSG", }, [NLMPROC_LOCK_MSG] = { .pc_func = nlmsvc_proc_lock_msg, @@ -610,6 +617,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "LOCK_MSG", }, [NLMPROC_CANCEL_MSG] = { .pc_func = nlmsvc_proc_cancel_msg, @@ -618,6 +626,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "CANCEL_MSG", }, [NLMPROC_UNLOCK_MSG] = { .pc_func = nlmsvc_proc_unlock_msg, @@ -626,6 +635,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNLOCK_MSG", }, [NLMPROC_GRANTED_MSG] = { .pc_func = nlmsvc_proc_granted_msg, @@ -634,6 +644,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "GRANTED_MSG", }, [NLMPROC_TEST_RES] = { .pc_func = nlmsvc_proc_null, @@ -642,6 +653,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "TEST_RES", }, [NLMPROC_LOCK_RES] = { .pc_func = nlmsvc_proc_null, @@ -650,6 +662,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "LOCK_RES", }, [NLMPROC_CANCEL_RES] = { .pc_func = nlmsvc_proc_null, @@ -658,6 +671,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "CANCEL_RES", }, [NLMPROC_UNLOCK_RES] = { .pc_func = nlmsvc_proc_null, @@ -666,6 +680,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNLOCK_RES", }, [NLMPROC_GRANTED_RES] = { .pc_func = nlmsvc_proc_granted_res, @@ -674,6 +689,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "GRANTED_RES", }, [NLMPROC_NSM_NOTIFY] = { .pc_func = nlmsvc_proc_sm_notify, @@ -682,6 +698,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_reboot), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "SM_NOTIFY", }, [17] = { .pc_func = nlmsvc_proc_unused, @@ -690,6 +707,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNUSED", }, [18] = { .pc_func = nlmsvc_proc_unused, @@ -698,6 +716,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNUSED", }, [19] = { .pc_func = nlmsvc_proc_unused, @@ -706,6 +725,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, + .pc_name = "UNUSED", }, [NLMPROC_SHARE] = { .pc_func = nlmsvc_proc_share, @@ -714,6 +734,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+1, + .pc_name = "SHARE", }, [NLMPROC_UNSHARE] = { .pc_func = nlmsvc_proc_unshare, @@ -722,6 +743,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+1, + .pc_name = "UNSHARE", }, [NLMPROC_NM_LOCK] = { .pc_func = nlmsvc_proc_nm_lock, @@ -730,6 +752,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, + .pc_name = "NM_LOCK", }, [NLMPROC_FREE_ALL] = { .pc_func = nlmsvc_proc_free_all, @@ -738,5 +761,6 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_argsize = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = 0, + .pc_name = "FREE_ALL", }, }; diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c index 79ff172eb1c8..c5348ba81129 100644 --- a/fs/nfs/callback_xdr.c +++ b/fs/nfs/callback_xdr.c @@ -1060,6 +1060,7 @@ static const struct svc_procedure nfs4_callback_procedures1[] = { .pc_decode = nfs4_decode_void, .pc_encode = nfs4_encode_void, .pc_xdrressize = 1, + .pc_name = "NULL", }, [CB_COMPOUND] = { .pc_func = nfs4_callback_compound, @@ -1067,6 +1068,7 @@ static const struct svc_procedure nfs4_callback_procedures1[] = { .pc_argsize = 256, .pc_ressize = 256, .pc_xdrressize = NFS4_CALLBACK_BUFSIZE, + .pc_name = "COMPOUND", } }; diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index b0f66604532a..899762da23c9 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -371,6 +371,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST, + .pc_name = "NULL", }, [ACLPROC2_GETACL] = { .pc_func = nfsacld_proc_getacl, @@ -381,6 +382,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_ressize = sizeof(struct nfsd3_getaclres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+1+2*(1+ACL), + .pc_name = "GETACL", }, [ACLPROC2_SETACL] = { .pc_func = nfsacld_proc_setacl, @@ -391,6 +393,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, + .pc_name = "SETACL", }, [ACLPROC2_GETATTR] = { .pc_func = nfsacld_proc_getattr, @@ -401,6 +404,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, + .pc_name = "GETATTR", }, [ACLPROC2_ACCESS] = { .pc_func = nfsacld_proc_access, @@ -411,6 +415,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_ressize = sizeof(struct nfsd3_accessres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT+1, + .pc_name = "SETATTR", }, }; diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c index 7c30876a31a1..9e1a92fb9771 100644 --- a/fs/nfsd/nfs3acl.c +++ b/fs/nfsd/nfs3acl.c @@ -251,6 +251,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST, + .pc_name = "NULL", }, [ACLPROC3_GETACL] = { .pc_func = nfsd3_proc_getacl, @@ -261,6 +262,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { .pc_ressize = sizeof(struct nfsd3_getaclres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+1+2*(1+ACL), + .pc_name = "GETACL", }, [ACLPROC3_SETACL] = { .pc_func = nfsd3_proc_setacl, @@ -271,6 +273,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { .pc_ressize = sizeof(struct nfsd3_attrstat), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT, + .pc_name = "SETACL", }, }; diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index 76931f4f57c3..c9c64471c568 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -708,6 +708,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST, + .pc_name = "NULL", }, [NFS3PROC_GETATTR] = { .pc_func = nfsd3_proc_getattr, @@ -718,6 +719,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_attrstatres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, + .pc_name = "GETATTR", }, [NFS3PROC_SETATTR] = { .pc_func = nfsd3_proc_setattr, @@ -728,6 +730,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_wccstatres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC, + .pc_name = "SETATTR", }, [NFS3PROC_LOOKUP] = { .pc_func = nfsd3_proc_lookup, @@ -738,6 +741,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_diropres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+FH+pAT+pAT, + .pc_name = "LOOKUP", }, [NFS3PROC_ACCESS] = { .pc_func = nfsd3_proc_access, @@ -748,6 +752,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_accessres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+1, + .pc_name = "ACCESS", }, [NFS3PROC_READLINK] = { .pc_func = nfsd3_proc_readlink, @@ -758,6 +763,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_readlinkres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+1+NFS3_MAXPATHLEN/4, + .pc_name = "READLINK", }, [NFS3PROC_READ] = { .pc_func = nfsd3_proc_read, @@ -768,6 +774,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_readres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+4+NFSSVC_MAXBLKSIZE/4, + .pc_name = "READ", }, [NFS3PROC_WRITE] = { .pc_func = nfsd3_proc_write, @@ -778,6 +785,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_writeres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC+4, + .pc_name = "WRITE", }, [NFS3PROC_CREATE] = { .pc_func = nfsd3_proc_create, @@ -788,6 +796,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_createres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+(1+FH+pAT)+WC, + .pc_name = "CREATE", }, [NFS3PROC_MKDIR] = { .pc_func = nfsd3_proc_mkdir, @@ -798,6 +807,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_createres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+(1+FH+pAT)+WC, + .pc_name = "MKDIR", }, [NFS3PROC_SYMLINK] = { .pc_func = nfsd3_proc_symlink, @@ -808,6 +818,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_createres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+(1+FH+pAT)+WC, + .pc_name = "SYMLINK", }, [NFS3PROC_MKNOD] = { .pc_func = nfsd3_proc_mknod, @@ -818,6 +829,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_createres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+(1+FH+pAT)+WC, + .pc_name = "MKNOD", }, [NFS3PROC_REMOVE] = { .pc_func = nfsd3_proc_remove, @@ -828,6 +840,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_wccstatres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC, + .pc_name = "REMOVE", }, [NFS3PROC_RMDIR] = { .pc_func = nfsd3_proc_rmdir, @@ -838,6 +851,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_wccstatres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC, + .pc_name = "RMDIR", }, [NFS3PROC_RENAME] = { .pc_func = nfsd3_proc_rename, @@ -848,6 +862,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_renameres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC+WC, + .pc_name = "RENAME", }, [NFS3PROC_LINK] = { .pc_func = nfsd3_proc_link, @@ -858,6 +873,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_linkres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+pAT+WC, + .pc_name = "LINK", }, [NFS3PROC_READDIR] = { .pc_func = nfsd3_proc_readdir, @@ -867,6 +883,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_argsize = sizeof(struct nfsd3_readdirargs), .pc_ressize = sizeof(struct nfsd3_readdirres), .pc_cachetype = RC_NOCACHE, + .pc_name = "READDIR", }, [NFS3PROC_READDIRPLUS] = { .pc_func = nfsd3_proc_readdirplus, @@ -876,6 +893,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_argsize = sizeof(struct nfsd3_readdirplusargs), .pc_ressize = sizeof(struct nfsd3_readdirres), .pc_cachetype = RC_NOCACHE, + .pc_name = "READDIRPLUS", }, [NFS3PROC_FSSTAT] = { .pc_func = nfsd3_proc_fsstat, @@ -885,6 +903,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_fsstatres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+2*6+1, + .pc_name = "FSSTAT", }, [NFS3PROC_FSINFO] = { .pc_func = nfsd3_proc_fsinfo, @@ -894,6 +913,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_fsinfores), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+12, + .pc_name = "FSINFO", }, [NFS3PROC_PATHCONF] = { .pc_func = nfsd3_proc_pathconf, @@ -903,6 +923,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_pathconfres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+6, + .pc_name = "PATHCONF", }, [NFS3PROC_COMMIT] = { .pc_func = nfsd3_proc_commit, @@ -913,6 +934,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_ressize = sizeof(struct nfsd3_commitres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+WC+2, + .pc_name = "COMMIT", }, }; diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 8d6d2678abad..f567592692ee 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -3305,6 +3305,7 @@ static const struct svc_procedure nfsd_procedures4[2] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = 1, + .pc_name = "NULL", }, [NFSPROC4_COMPOUND] = { .pc_func = nfsd4_proc_compound, @@ -3315,6 +3316,7 @@ static const struct svc_procedure nfsd_procedures4[2] = { .pc_release = nfsd4_release_compoundargs, .pc_cachetype = RC_NOCACHE, .pc_xdrressize = NFSD_BUFSIZE/4, + .pc_name = "COMPOUND", }, }; diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 9473d048efec..1f85a4dc9d1b 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -623,6 +623,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = 0, + .pc_name = "NULL", }, [NFSPROC_GETATTR] = { .pc_func = nfsd_proc_getattr, @@ -633,6 +634,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, + .pc_name = "GETATTR", }, [NFSPROC_SETATTR] = { .pc_func = nfsd_proc_setattr, @@ -643,6 +645,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+AT, + .pc_name = "SETATTR", }, [NFSPROC_ROOT] = { .pc_func = nfsd_proc_root, @@ -652,6 +655,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = 0, + .pc_name = "ROOT", }, [NFSPROC_LOOKUP] = { .pc_func = nfsd_proc_lookup, @@ -662,6 +666,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_diropres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+FH+AT, + .pc_name = "LOOKUP", }, [NFSPROC_READLINK] = { .pc_func = nfsd_proc_readlink, @@ -671,6 +676,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_readlinkres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+1+NFS_MAXPATHLEN/4, + .pc_name = "READLINK", }, [NFSPROC_READ] = { .pc_func = nfsd_proc_read, @@ -681,6 +687,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_readres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT+1+NFSSVC_MAXBLKSIZE_V2/4, + .pc_name = "READ", }, [NFSPROC_WRITECACHE] = { .pc_func = nfsd_proc_writecache, @@ -690,6 +697,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = 0, + .pc_name = "WRITECACHE", }, [NFSPROC_WRITE] = { .pc_func = nfsd_proc_write, @@ -700,6 +708,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+AT, + .pc_name = "WRITE", }, [NFSPROC_CREATE] = { .pc_func = nfsd_proc_create, @@ -710,6 +719,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_diropres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+FH+AT, + .pc_name = "CREATE", }, [NFSPROC_REMOVE] = { .pc_func = nfsd_proc_remove, @@ -719,6 +729,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, + .pc_name = "REMOVE", }, [NFSPROC_RENAME] = { .pc_func = nfsd_proc_rename, @@ -728,6 +739,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, + .pc_name = "RENAME", }, [NFSPROC_LINK] = { .pc_func = nfsd_proc_link, @@ -737,6 +749,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, + .pc_name = "LINK", }, [NFSPROC_SYMLINK] = { .pc_func = nfsd_proc_symlink, @@ -746,6 +759,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, + .pc_name = "SYMLINK", }, [NFSPROC_MKDIR] = { .pc_func = nfsd_proc_mkdir, @@ -756,6 +770,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_diropres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+FH+AT, + .pc_name = "MKDIR", }, [NFSPROC_RMDIR] = { .pc_func = nfsd_proc_rmdir, @@ -765,6 +780,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, + .pc_name = "RMDIR", }, [NFSPROC_READDIR] = { .pc_func = nfsd_proc_readdir, @@ -773,6 +789,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_argsize = sizeof(struct nfsd_readdirargs), .pc_ressize = sizeof(struct nfsd_readdirres), .pc_cachetype = RC_NOCACHE, + .pc_name = "READDIR", }, [NFSPROC_STATFS] = { .pc_func = nfsd_proc_statfs, @@ -782,6 +799,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_ressize = sizeof(struct nfsd_statfsres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+5, + .pc_name = "STATFS", }, }; diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 34c2a69820e9..31ee3b6047c3 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -463,6 +463,7 @@ struct svc_procedure { unsigned int pc_ressize; /* result struct size */ unsigned int pc_cachetype; /* cache info (NFS) */ unsigned int pc_xdrressize; /* maximum size of XDR reply */ + const char * pc_name; /* for display */ }; /* -- cgit v1.2.3 From 81d217474326b25d7f14274b02fe3da1e85ad934 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 27 Nov 2020 17:37:02 -0500 Subject: SUNRPC: Move definition of XDR_UNIT Clean up: The unit of XDR alignment is defined by RFC 4506, not as part of the RPC message header. Thus it belongs in include/linux/sunrpc/xdr.h. Signed-off-by: Chuck Lever --- include/linux/sunrpc/msg_prot.h | 3 --- include/linux/sunrpc/xdr.h | 13 ++++++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/msg_prot.h b/include/linux/sunrpc/msg_prot.h index 43f854487539..938c2bf29db8 100644 --- a/include/linux/sunrpc/msg_prot.h +++ b/include/linux/sunrpc/msg_prot.h @@ -10,9 +10,6 @@ #define RPC_VERSION 2 -/* size of an XDR encoding unit in bytes, i.e. 32bit */ -#define XDR_UNIT (4) - /* spec defines authentication flavor as an unsigned 32 bit integer */ typedef u32 rpc_authflavor_t; diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index 19b6dea27367..dbba537caab6 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -19,6 +19,13 @@ struct bio_vec; struct rpc_rqst; +/* + * Size of an XDR encoding unit in bytes, i.e. 32 bits, + * as defined in Section 3 of RFC 4506. All encoded + * XDR data items are aligned on a boundary of 32 bits. + */ +#define XDR_UNIT sizeof(__be32) + /* * Buffer adjustment */ @@ -330,7 +337,7 @@ ssize_t xdr_stream_decode_string_dup(struct xdr_stream *xdr, char **str, static inline size_t xdr_align_size(size_t n) { - const size_t mask = sizeof(__u32) - 1; + const size_t mask = XDR_UNIT - 1; return (n + mask) & ~mask; } @@ -360,7 +367,7 @@ static inline size_t xdr_pad_size(size_t n) */ static inline ssize_t xdr_stream_encode_item_present(struct xdr_stream *xdr) { - const size_t len = sizeof(__be32); + const size_t len = XDR_UNIT; __be32 *p = xdr_reserve_space(xdr, len); if (unlikely(!p)) @@ -379,7 +386,7 @@ static inline ssize_t xdr_stream_encode_item_present(struct xdr_stream *xdr) */ static inline int xdr_stream_encode_item_absent(struct xdr_stream *xdr) { - const size_t len = sizeof(__be32); + const size_t len = XDR_UNIT; __be32 *p = xdr_reserve_space(xdr, len); if (unlikely(!p)) -- cgit v1.2.3 From 6bb844b4eb6e3b109a2fdaffb60e6da722dc4356 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 17 Nov 2020 10:38:46 -0500 Subject: NFSD: Add an xdr_stream-based decoder for NFSv2/3 ACLs Signed-off-by: Chuck Lever --- fs/nfs_common/nfsacl.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/nfsacl.h | 3 +++ 2 files changed, 55 insertions(+) (limited to 'include/linux') diff --git a/fs/nfs_common/nfsacl.c b/fs/nfs_common/nfsacl.c index d056ad2fdefd..79c563c1a5e8 100644 --- a/fs/nfs_common/nfsacl.c +++ b/fs/nfs_common/nfsacl.c @@ -295,3 +295,55 @@ int nfsacl_decode(struct xdr_buf *buf, unsigned int base, unsigned int *aclcnt, nfsacl_desc.desc.array_len; } EXPORT_SYMBOL_GPL(nfsacl_decode); + +/** + * nfs_stream_decode_acl - Decode an NFSv3 ACL + * + * @xdr: an xdr_stream positioned at an encoded ACL + * @aclcnt: OUT: count of ACEs in decoded posix_acl + * @pacl: OUT: a dynamically-allocated buffer containing the decoded posix_acl + * + * Return values: + * %false: The encoded ACL is not valid + * %true: @pacl contains a decoded ACL, and @xdr is advanced + * + * On a successful return, caller must release *pacl using posix_acl_release(). + */ +bool nfs_stream_decode_acl(struct xdr_stream *xdr, unsigned int *aclcnt, + struct posix_acl **pacl) +{ + const size_t elem_size = XDR_UNIT * 3; + struct nfsacl_decode_desc nfsacl_desc = { + .desc = { + .elem_size = elem_size, + .xcode = pacl ? xdr_nfsace_decode : NULL, + }, + }; + unsigned int base; + u32 entries; + + if (xdr_stream_decode_u32(xdr, &entries) < 0) + return false; + if (entries > NFS_ACL_MAX_ENTRIES) + return false; + + base = xdr_stream_pos(xdr); + if (!xdr_inline_decode(xdr, XDR_UNIT + elem_size * entries)) + return false; + nfsacl_desc.desc.array_maxlen = entries; + if (xdr_decode_array2(xdr->buf, base, &nfsacl_desc.desc)) + return false; + + if (pacl) { + if (entries != nfsacl_desc.desc.array_len || + posix_acl_from_nfsacl(nfsacl_desc.acl) != 0) { + posix_acl_release(nfsacl_desc.acl); + return false; + } + *pacl = nfsacl_desc.acl; + } + if (aclcnt) + *aclcnt = entries; + return true; +} +EXPORT_SYMBOL_GPL(nfs_stream_decode_acl); diff --git a/include/linux/nfsacl.h b/include/linux/nfsacl.h index 103d44695323..0ba99c513649 100644 --- a/include/linux/nfsacl.h +++ b/include/linux/nfsacl.h @@ -38,5 +38,8 @@ nfsacl_encode(struct xdr_buf *buf, unsigned int base, struct inode *inode, extern int nfsacl_decode(struct xdr_buf *buf, unsigned int base, unsigned int *aclcnt, struct posix_acl **pacl); +extern bool +nfs_stream_decode_acl(struct xdr_stream *xdr, unsigned int *aclcnt, + struct posix_acl **pacl); #endif /* __LINUX_NFSACL_H */ -- cgit v1.2.3 From df971cd853c05778ae1175e8aeb80a04bb9d4be5 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Dec 2020 15:47:44 -0500 Subject: svcrdma: Convert rdma_stat_recv to a per-CPU counter Receives are frequent events. Avoid the overhead of a memory bus lock cycle for counting a value that is hardly every used. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 3 +- net/sunrpc/xprtrdma/svc_rdma.c | 55 ++++++++++++++++++++++++++++++--- net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 3 +- 3 files changed, 54 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index 294b56e61522..ff32c59a27e7 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -49,6 +49,7 @@ #include #include +#include #include #include @@ -65,7 +66,7 @@ extern unsigned int svcrdma_max_requests; extern unsigned int svcrdma_max_bc_requests; extern unsigned int svcrdma_max_req_size; -extern atomic_t rdma_stat_recv; +extern struct percpu_counter svcrdma_stat_recv; extern atomic_t rdma_stat_read; extern atomic_t rdma_stat_write; extern atomic_t rdma_stat_sq_starve; diff --git a/net/sunrpc/xprtrdma/svc_rdma.c b/net/sunrpc/xprtrdma/svc_rdma.c index 1fc1d5cbeb9b..3e5e622bad81 100644 --- a/net/sunrpc/xprtrdma/svc_rdma.c +++ b/net/sunrpc/xprtrdma/svc_rdma.c @@ -63,7 +63,7 @@ unsigned int svcrdma_max_req_size = RPCRDMA_DEF_INLINE_THRESH; static unsigned int min_max_inline = RPCRDMA_DEF_INLINE_THRESH; static unsigned int max_max_inline = RPCRDMA_MAX_INLINE_THRESH; -atomic_t rdma_stat_recv; +struct percpu_counter svcrdma_stat_recv; atomic_t rdma_stat_read; atomic_t rdma_stat_write; atomic_t rdma_stat_sq_starve; @@ -110,6 +110,42 @@ static int read_reset_stat(struct ctl_table *table, int write, return 0; } +enum { + SVCRDMA_COUNTER_BUFSIZ = sizeof(unsigned long long), +}; + +static int svcrdma_counter_handler(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + struct percpu_counter *stat = (struct percpu_counter *)table->data; + char tmp[SVCRDMA_COUNTER_BUFSIZ + 1]; + int len; + + if (write) { + percpu_counter_set(stat, 0); + return 0; + } + + len = snprintf(tmp, SVCRDMA_COUNTER_BUFSIZ, "%lld\n", + percpu_counter_sum_positive(stat)); + if (len >= SVCRDMA_COUNTER_BUFSIZ) + return -EFAULT; + len = strlen(tmp); + if (*ppos > len) { + *lenp = 0; + return 0; + } + len -= *ppos; + if (len > *lenp) + len = *lenp; + if (len) + memcpy(buffer, tmp, len); + *lenp = len; + *ppos += len; + + return 0; +} + static struct ctl_table_header *svcrdma_table_header; static struct ctl_table svcrdma_parm_table[] = { { @@ -149,10 +185,10 @@ static struct ctl_table svcrdma_parm_table[] = { }, { .procname = "rdma_stat_recv", - .data = &rdma_stat_recv, - .maxlen = sizeof(atomic_t), + .data = &svcrdma_stat_recv, + .maxlen = SVCRDMA_COUNTER_BUFSIZ, .mode = 0644, - .proc_handler = read_reset_stat, + .proc_handler = svcrdma_counter_handler, }, { .procname = "rdma_stat_write", @@ -230,15 +266,26 @@ static void svc_rdma_proc_cleanup(void) return; unregister_sysctl_table(svcrdma_table_header); svcrdma_table_header = NULL; + + percpu_counter_destroy(&svcrdma_stat_recv); } static int svc_rdma_proc_init(void) { + int rc; + if (svcrdma_table_header) return 0; + rc = percpu_counter_init(&svcrdma_stat_recv, 0, GFP_KERNEL); + if (rc) + goto out_err; + svcrdma_table_header = register_sysctl_table(svcrdma_root_table); return 0; + +out_err: + return rc; } void svc_rdma_cleanup(void) diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c index cbdb71247755..7d14a74df716 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c @@ -845,8 +845,7 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp) } list_del(&ctxt->rc_list); spin_unlock(&rdma_xprt->sc_rq_dto_lock); - - atomic_inc(&rdma_stat_recv); + percpu_counter_inc(&svcrdma_stat_recv); svc_rdma_build_arg_xdr(rqstp, ctxt); -- cgit v1.2.3 From 22df5a22462e836ccb30634c3a52602091179a73 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Dec 2020 15:55:17 -0500 Subject: svcrdma: Convert rdma_stat_sq_starve to a per-CPU counter Avoid the overhead of a memory bus lock cycle for counting a value that is hardly every used. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 2 +- net/sunrpc/xprtrdma/svc_rdma.c | 13 +++++++++---- net/sunrpc/xprtrdma/svc_rdma_rw.c | 1 + net/sunrpc/xprtrdma/svc_rdma_sendto.c | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index ff32c59a27e7..c06b16ccf83e 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -69,7 +69,7 @@ extern unsigned int svcrdma_max_req_size; extern struct percpu_counter svcrdma_stat_recv; extern atomic_t rdma_stat_read; extern atomic_t rdma_stat_write; -extern atomic_t rdma_stat_sq_starve; +extern struct percpu_counter svcrdma_stat_sq_starve; extern atomic_t rdma_stat_rq_starve; extern atomic_t rdma_stat_rq_poll; extern atomic_t rdma_stat_rq_prod; diff --git a/net/sunrpc/xprtrdma/svc_rdma.c b/net/sunrpc/xprtrdma/svc_rdma.c index 3e5e622bad81..ee768d4c3c90 100644 --- a/net/sunrpc/xprtrdma/svc_rdma.c +++ b/net/sunrpc/xprtrdma/svc_rdma.c @@ -66,7 +66,7 @@ static unsigned int max_max_inline = RPCRDMA_MAX_INLINE_THRESH; struct percpu_counter svcrdma_stat_recv; atomic_t rdma_stat_read; atomic_t rdma_stat_write; -atomic_t rdma_stat_sq_starve; +struct percpu_counter svcrdma_stat_sq_starve; atomic_t rdma_stat_rq_starve; atomic_t rdma_stat_rq_poll; atomic_t rdma_stat_rq_prod; @@ -199,10 +199,10 @@ static struct ctl_table svcrdma_parm_table[] = { }, { .procname = "rdma_stat_sq_starve", - .data = &rdma_stat_sq_starve, - .maxlen = sizeof(atomic_t), + .data = &svcrdma_stat_sq_starve, + .maxlen = SVCRDMA_COUNTER_BUFSIZ, .mode = 0644, - .proc_handler = read_reset_stat, + .proc_handler = svcrdma_counter_handler, }, { .procname = "rdma_stat_rq_starve", @@ -267,6 +267,7 @@ static void svc_rdma_proc_cleanup(void) unregister_sysctl_table(svcrdma_table_header); svcrdma_table_header = NULL; + percpu_counter_destroy(&svcrdma_stat_sq_starve); percpu_counter_destroy(&svcrdma_stat_recv); } @@ -278,6 +279,9 @@ static int svc_rdma_proc_init(void) return 0; rc = percpu_counter_init(&svcrdma_stat_recv, 0, GFP_KERNEL); + if (rc) + goto out_err; + rc = percpu_counter_init(&svcrdma_stat_sq_starve, 0, GFP_KERNEL); if (rc) goto out_err; @@ -285,6 +289,7 @@ static int svc_rdma_proc_init(void) return 0; out_err: + percpu_counter_destroy(&svcrdma_stat_recv); return rc; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index 0b63e1321d74..d7d98b2df00b 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -364,6 +364,7 @@ static int svc_rdma_post_chunk_ctxt(struct svc_rdma_chunk_ctxt *cc) return 0; } + percpu_counter_inc(&svcrdma_stat_sq_starve); trace_svcrdma_sq_full(rdma); atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); wait_event(rdma->sc_send_wait, diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index 68af79d4f04f..52c759a8543e 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -317,7 +317,7 @@ int svc_rdma_send(struct svcxprt_rdma *rdma, struct svc_rdma_send_ctxt *ctxt) /* If the SQ is full, wait until an SQ entry is available */ while (1) { if ((atomic_dec_return(&rdma->sc_sq_avail) < 0)) { - atomic_inc(&rdma_stat_sq_starve); + percpu_counter_inc(&svcrdma_stat_sq_starve); trace_svcrdma_sq_full(rdma); atomic_inc(&rdma->sc_sq_avail); wait_event(rdma->sc_send_wait, -- cgit v1.2.3 From 1e7e55731628c90d8c701c45f9c3a3b8718840d6 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Dec 2020 16:09:36 -0500 Subject: svcrdma: Restore read and write stats Now that we have an efficient mechanism to update these two stats, let's start maintaining them again. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 4 ++-- net/sunrpc/xprtrdma/svc_rdma.c | 26 ++++++++++++++++++-------- net/sunrpc/xprtrdma/svc_rdma_rw.c | 2 ++ 3 files changed, 22 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index c06b16ccf83e..c882816cba8e 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -66,10 +66,10 @@ extern unsigned int svcrdma_max_requests; extern unsigned int svcrdma_max_bc_requests; extern unsigned int svcrdma_max_req_size; +extern struct percpu_counter svcrdma_stat_read; extern struct percpu_counter svcrdma_stat_recv; -extern atomic_t rdma_stat_read; -extern atomic_t rdma_stat_write; extern struct percpu_counter svcrdma_stat_sq_starve; +extern struct percpu_counter svcrdma_stat_write; extern atomic_t rdma_stat_rq_starve; extern atomic_t rdma_stat_rq_poll; extern atomic_t rdma_stat_rq_prod; diff --git a/net/sunrpc/xprtrdma/svc_rdma.c b/net/sunrpc/xprtrdma/svc_rdma.c index ee768d4c3c90..c8336df7a142 100644 --- a/net/sunrpc/xprtrdma/svc_rdma.c +++ b/net/sunrpc/xprtrdma/svc_rdma.c @@ -63,10 +63,10 @@ unsigned int svcrdma_max_req_size = RPCRDMA_DEF_INLINE_THRESH; static unsigned int min_max_inline = RPCRDMA_DEF_INLINE_THRESH; static unsigned int max_max_inline = RPCRDMA_MAX_INLINE_THRESH; +struct percpu_counter svcrdma_stat_read; struct percpu_counter svcrdma_stat_recv; -atomic_t rdma_stat_read; -atomic_t rdma_stat_write; struct percpu_counter svcrdma_stat_sq_starve; +struct percpu_counter svcrdma_stat_write; atomic_t rdma_stat_rq_starve; atomic_t rdma_stat_rq_poll; atomic_t rdma_stat_rq_prod; @@ -178,10 +178,10 @@ static struct ctl_table svcrdma_parm_table[] = { { .procname = "rdma_stat_read", - .data = &rdma_stat_read, - .maxlen = sizeof(atomic_t), + .data = &svcrdma_stat_read, + .maxlen = SVCRDMA_COUNTER_BUFSIZ, .mode = 0644, - .proc_handler = read_reset_stat, + .proc_handler = svcrdma_counter_handler, }, { .procname = "rdma_stat_recv", @@ -192,10 +192,10 @@ static struct ctl_table svcrdma_parm_table[] = { }, { .procname = "rdma_stat_write", - .data = &rdma_stat_write, - .maxlen = sizeof(atomic_t), + .data = &svcrdma_stat_write, + .maxlen = SVCRDMA_COUNTER_BUFSIZ, .mode = 0644, - .proc_handler = read_reset_stat, + .proc_handler = svcrdma_counter_handler, }, { .procname = "rdma_stat_sq_starve", @@ -267,8 +267,10 @@ static void svc_rdma_proc_cleanup(void) unregister_sysctl_table(svcrdma_table_header); svcrdma_table_header = NULL; + percpu_counter_destroy(&svcrdma_stat_write); percpu_counter_destroy(&svcrdma_stat_sq_starve); percpu_counter_destroy(&svcrdma_stat_recv); + percpu_counter_destroy(&svcrdma_stat_read); } static int svc_rdma_proc_init(void) @@ -278,10 +280,16 @@ static int svc_rdma_proc_init(void) if (svcrdma_table_header) return 0; + rc = percpu_counter_init(&svcrdma_stat_read, 0, GFP_KERNEL); + if (rc) + goto out_err; rc = percpu_counter_init(&svcrdma_stat_recv, 0, GFP_KERNEL); if (rc) goto out_err; rc = percpu_counter_init(&svcrdma_stat_sq_starve, 0, GFP_KERNEL); + if (rc) + goto out_err; + rc = percpu_counter_init(&svcrdma_stat_write, 0, GFP_KERNEL); if (rc) goto out_err; @@ -289,7 +297,9 @@ static int svc_rdma_proc_init(void) return 0; out_err: + percpu_counter_destroy(&svcrdma_stat_sq_starve); percpu_counter_destroy(&svcrdma_stat_recv); + percpu_counter_destroy(&svcrdma_stat_read); return rc; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index d7d98b2df00b..693d139a8633 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -469,6 +469,7 @@ svc_rdma_build_writes(struct svc_rdma_write_info *info, DMA_TO_DEVICE); if (ret < 0) return -EIO; + percpu_counter_inc(&svcrdma_stat_write); list_add(&ctxt->rw_list, &cc->cc_rwctxts); cc->cc_sqecount += ret; @@ -719,6 +720,7 @@ static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info, segment->rs_handle, DMA_FROM_DEVICE); if (ret < 0) return -EIO; + percpu_counter_inc(&svcrdma_stat_read); list_add(&ctxt->rw_list, &cc->cc_rwctxts); cc->cc_sqecount += ret; -- cgit v1.2.3 From c6226ff9a62a17182b8092883ca201df5cd47f59 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 29 Dec 2020 14:53:09 -0500 Subject: svcrdma: Deprecate stat variables that are no longer used Clean up. We are not permitted to remove old proc files. Instead, convert these variables to stubs that are only ever allowed to display a value of zero. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 5 --- net/sunrpc/xprtrdma/svc_rdma.c | 84 +++++++++++++---------------------------- 2 files changed, 27 insertions(+), 62 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index c882816cba8e..1e76ed688044 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -70,11 +70,6 @@ extern struct percpu_counter svcrdma_stat_read; extern struct percpu_counter svcrdma_stat_recv; extern struct percpu_counter svcrdma_stat_sq_starve; extern struct percpu_counter svcrdma_stat_write; -extern atomic_t rdma_stat_rq_starve; -extern atomic_t rdma_stat_rq_poll; -extern atomic_t rdma_stat_rq_prod; -extern atomic_t rdma_stat_sq_poll; -extern atomic_t rdma_stat_sq_prod; struct svcxprt_rdma { struct svc_xprt sc_xprt; /* SVC transport structure */ diff --git a/net/sunrpc/xprtrdma/svc_rdma.c b/net/sunrpc/xprtrdma/svc_rdma.c index c8336df7a142..5bc20e9d09cd 100644 --- a/net/sunrpc/xprtrdma/svc_rdma.c +++ b/net/sunrpc/xprtrdma/svc_rdma.c @@ -62,53 +62,13 @@ static unsigned int max_max_requests = 16384; unsigned int svcrdma_max_req_size = RPCRDMA_DEF_INLINE_THRESH; static unsigned int min_max_inline = RPCRDMA_DEF_INLINE_THRESH; static unsigned int max_max_inline = RPCRDMA_MAX_INLINE_THRESH; +static unsigned int svcrdma_stat_unused; +static unsigned int zero; struct percpu_counter svcrdma_stat_read; struct percpu_counter svcrdma_stat_recv; struct percpu_counter svcrdma_stat_sq_starve; struct percpu_counter svcrdma_stat_write; -atomic_t rdma_stat_rq_starve; -atomic_t rdma_stat_rq_poll; -atomic_t rdma_stat_rq_prod; -atomic_t rdma_stat_sq_poll; -atomic_t rdma_stat_sq_prod; - -/* - * This function implements reading and resetting an atomic_t stat - * variable through read/write to a proc file. Any write to the file - * resets the associated statistic to zero. Any read returns it's - * current value. - */ -static int read_reset_stat(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - atomic_t *stat = (atomic_t *)table->data; - - if (!stat) - return -EINVAL; - - if (write) - atomic_set(stat, 0); - else { - char str_buf[32]; - int len = snprintf(str_buf, 32, "%d\n", atomic_read(stat)); - if (len >= 32) - return -EFAULT; - len = strlen(str_buf); - if (*ppos > len) { - *lenp = 0; - return 0; - } - len -= *ppos; - if (len > *lenp) - len = *lenp; - if (len) - memcpy(buffer, str_buf, len); - *lenp = len; - *ppos += len; - } - return 0; -} enum { SVCRDMA_COUNTER_BUFSIZ = sizeof(unsigned long long), @@ -206,38 +166,48 @@ static struct ctl_table svcrdma_parm_table[] = { }, { .procname = "rdma_stat_rq_starve", - .data = &rdma_stat_rq_starve, - .maxlen = sizeof(atomic_t), + .data = &svcrdma_stat_unused, + .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = read_reset_stat, + .proc_handler = proc_dointvec_minmax, + .extra1 = &zero, + .extra2 = &zero, }, { .procname = "rdma_stat_rq_poll", - .data = &rdma_stat_rq_poll, - .maxlen = sizeof(atomic_t), + .data = &svcrdma_stat_unused, + .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = read_reset_stat, + .proc_handler = proc_dointvec_minmax, + .extra1 = &zero, + .extra2 = &zero, }, { .procname = "rdma_stat_rq_prod", - .data = &rdma_stat_rq_prod, - .maxlen = sizeof(atomic_t), + .data = &svcrdma_stat_unused, + .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = read_reset_stat, + .proc_handler = proc_dointvec_minmax, + .extra1 = &zero, + .extra2 = &zero, }, { .procname = "rdma_stat_sq_poll", - .data = &rdma_stat_sq_poll, - .maxlen = sizeof(atomic_t), + .data = &svcrdma_stat_unused, + .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = read_reset_stat, + .proc_handler = proc_dointvec_minmax, + .extra1 = &zero, + .extra2 = &zero, }, { .procname = "rdma_stat_sq_prod", - .data = &rdma_stat_sq_prod, - .maxlen = sizeof(atomic_t), + .data = &svcrdma_stat_unused, + .maxlen = sizeof(unsigned int), .mode = 0644, - .proc_handler = read_reset_stat, + .proc_handler = proc_dointvec_minmax, + .extra1 = &zero, + .extra2 = &zero, }, { }, }; -- cgit v1.2.3 From 43042b90cae11cc2d9827c91df6d6b5fe498d5ce Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 8 Dec 2020 13:14:15 -0500 Subject: svcrdma: Reduce Receive doorbell rate This is similar to commit e340c2d6ef2a ("xprtrdma: Reduce the doorbell rate (Receive)") which added Receive batching to the client. Signed-off-by: Chuck Lever --- include/linux/sunrpc/svc_rdma.h | 1 + net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 82 +++++++++++++++++---------------- 2 files changed, 44 insertions(+), 39 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index 1e76ed688044..7c693b31965e 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -104,6 +104,7 @@ struct svcxprt_rdma { wait_queue_head_t sc_send_wait; /* SQ exhaustion waitlist */ unsigned long sc_flags; + u32 sc_pending_recvs; struct list_head sc_read_complete_q; struct work_struct sc_work; diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c index 7d14a74df716..ab0b7e9777bc 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c @@ -266,33 +266,46 @@ void svc_rdma_release_rqst(struct svc_rqst *rqstp) svc_rdma_recv_ctxt_put(rdma, ctxt); } -static int __svc_rdma_post_recv(struct svcxprt_rdma *rdma, - struct svc_rdma_recv_ctxt *ctxt) +static bool svc_rdma_refresh_recvs(struct svcxprt_rdma *rdma, + unsigned int wanted, bool temp) { + const struct ib_recv_wr *bad_wr = NULL; + struct svc_rdma_recv_ctxt *ctxt; + struct ib_recv_wr *recv_chain; int ret; - trace_svcrdma_post_recv(ctxt); - ret = ib_post_recv(rdma->sc_qp, &ctxt->rc_recv_wr, NULL); + recv_chain = NULL; + while (wanted--) { + ctxt = svc_rdma_recv_ctxt_get(rdma); + if (!ctxt) + break; + + trace_svcrdma_post_recv(ctxt); + ctxt->rc_temp = temp; + ctxt->rc_recv_wr.next = recv_chain; + recv_chain = &ctxt->rc_recv_wr; + rdma->sc_pending_recvs++; + } + if (!recv_chain) + return false; + + ret = ib_post_recv(rdma->sc_qp, recv_chain, &bad_wr); if (ret) goto err_post; - return 0; + return true; err_post: - trace_svcrdma_rq_post_err(rdma, ret); - svc_rdma_recv_ctxt_put(rdma, ctxt); - return ret; -} - -static int svc_rdma_post_recv(struct svcxprt_rdma *rdma) -{ - struct svc_rdma_recv_ctxt *ctxt; + while (bad_wr) { + ctxt = container_of(bad_wr, struct svc_rdma_recv_ctxt, + rc_recv_wr); + bad_wr = bad_wr->next; + svc_rdma_recv_ctxt_put(rdma, ctxt); + } - if (test_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags)) - return 0; - ctxt = svc_rdma_recv_ctxt_get(rdma); - if (!ctxt) - return -ENOMEM; - return __svc_rdma_post_recv(rdma, ctxt); + trace_svcrdma_rq_post_err(rdma, ret); + /* Since we're destroying the xprt, no need to reset + * sc_pending_recvs. */ + return false; } /** @@ -303,20 +316,7 @@ static int svc_rdma_post_recv(struct svcxprt_rdma *rdma) */ bool svc_rdma_post_recvs(struct svcxprt_rdma *rdma) { - struct svc_rdma_recv_ctxt *ctxt; - unsigned int i; - int ret; - - for (i = 0; i < rdma->sc_max_requests; i++) { - ctxt = svc_rdma_recv_ctxt_get(rdma); - if (!ctxt) - return false; - ctxt->rc_temp = true; - ret = __svc_rdma_post_recv(rdma, ctxt); - if (ret) - return false; - } - return true; + return svc_rdma_refresh_recvs(rdma, rdma->sc_max_requests, true); } /** @@ -324,8 +324,6 @@ bool svc_rdma_post_recvs(struct svcxprt_rdma *rdma) * @cq: Completion Queue context * @wc: Work Completion object * - * NB: The svc_xprt/svcxprt_rdma is pinned whenever it's possible that - * the Receive completion handler could be running. */ static void svc_rdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) { @@ -333,6 +331,8 @@ static void svc_rdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) struct ib_cqe *cqe = wc->wr_cqe; struct svc_rdma_recv_ctxt *ctxt; + rdma->sc_pending_recvs--; + /* WARNING: Only wc->wr_cqe and wc->status are reliable */ ctxt = container_of(cqe, struct svc_rdma_recv_ctxt, rc_cqe); @@ -340,9 +340,6 @@ static void svc_rdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) if (wc->status != IB_WC_SUCCESS) goto flushed; - if (svc_rdma_post_recv(rdma)) - goto post_err; - /* All wc fields are now known to be valid */ ctxt->rc_byte_len = wc->byte_len; ib_dma_sync_single_for_cpu(rdma->sc_pd->device, @@ -356,11 +353,18 @@ static void svc_rdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) spin_unlock(&rdma->sc_rq_dto_lock); if (!test_bit(RDMAXPRT_CONN_PENDING, &rdma->sc_flags)) svc_xprt_enqueue(&rdma->sc_xprt); + + if (!test_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags) && + rdma->sc_pending_recvs < rdma->sc_max_requests) + if (!svc_rdma_refresh_recvs(rdma, RPCRDMA_MAX_RECV_BATCH, + false)) + goto post_err; + return; flushed: -post_err: svc_rdma_recv_ctxt_put(rdma, ctxt); +post_err: set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags); svc_xprt_enqueue(&rdma->sc_xprt); } -- cgit v1.2.3 From 3e1a88ec96259282b9a8b45c3f1fda7a3ff4f6ea Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 9 Jan 2021 16:03:02 +0000 Subject: bio: add a helper calculating nr segments to alloc Add a helper function calculating the number of bvec segments we need to allocate to construct a bio. It doesn't change anything functionally, but will be used to not duplicate special cases in the future. Reviewed-by: Christoph Hellwig Signed-off-by: Pavel Begunkov Reviewed-by: Ming Lei Signed-off-by: Jens Axboe --- fs/block_dev.c | 7 ++++--- fs/iomap/direct-io.c | 9 ++++----- include/linux/bio.h | 10 ++++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/fs/block_dev.c b/fs/block_dev.c index 3b8963e228a1..6f5bd9950baf 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -416,7 +416,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages) dio->size += bio->bi_iter.bi_size; pos += bio->bi_iter.bi_size; - nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES); + nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_PAGES); if (!nr_pages) { bool polled = false; @@ -481,9 +481,10 @@ blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) { int nr_pages; - nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES + 1); - if (!nr_pages) + if (!iov_iter_count(iter)) return 0; + + nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_PAGES + 1); if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES) return __blkdev_direct_IO_simple(iocb, iter, nr_pages); diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 933f234d5bec..ea1e8f696076 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -250,11 +250,8 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, orig_count = iov_iter_count(dio->submit.iter); iov_iter_truncate(dio->submit.iter, length); - nr_pages = iov_iter_npages(dio->submit.iter, BIO_MAX_PAGES); - if (nr_pages <= 0) { - ret = nr_pages; + if (!iov_iter_count(dio->submit.iter)) goto out; - } if (need_zeroout) { /* zero out from the start of the block to the write offset */ @@ -263,6 +260,7 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, iomap_dio_zero(dio, iomap, pos - pad, pad); } + nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_PAGES); do { size_t n; if (dio->error) { @@ -308,7 +306,8 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, dio->size += n; copied += n; - nr_pages = iov_iter_npages(dio->submit.iter, BIO_MAX_PAGES); + nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, + BIO_MAX_PAGES); iomap_dio_submit_bio(dio, iomap, bio, pos); pos += n; } while (nr_pages); diff --git a/include/linux/bio.h b/include/linux/bio.h index e135b500df5d..9ddb19801a03 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -10,6 +10,7 @@ #include /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */ #include +#include #define BIO_DEBUG @@ -441,6 +442,15 @@ static inline void bio_wouldblock_error(struct bio *bio) bio_endio(bio); } +/* + * Calculate number of bvec segments that should be allocated to fit data + * pointed by @iter. + */ +static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs) +{ + return iov_iter_npages(iter, max_segs); +} + struct request_queue; extern int submit_bio_wait(struct bio *bio); -- cgit v1.2.3 From c42bca92be928ce7dece5fc04cf68d0e37ee6718 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 9 Jan 2021 16:03:03 +0000 Subject: bio: don't copy bvec for direct IO The block layer spends quite a while in blkdev_direct_IO() to copy and initialise bio's bvec. However, if we've already got a bvec in the input iterator it might be reused in some cases, i.e. when new ITER_BVEC_FLAG_FIXED flag is set. Simple tests show considerable performance boost, and it also reduces memory footprint. Suggested-by: Matthew Wilcox Reviewed-by: Christoph Hellwig Signed-off-by: Pavel Begunkov Reviewed-by: Ming Lei Signed-off-by: Jens Axboe --- Documentation/filesystems/porting.rst | 9 +++++ block/bio.c | 67 +++++++++++++++-------------------- include/linux/bio.h | 5 ++- 3 files changed, 42 insertions(+), 39 deletions(-) (limited to 'include/linux') diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst index c722d94f29ea..1f8cf8e10b34 100644 --- a/Documentation/filesystems/porting.rst +++ b/Documentation/filesystems/porting.rst @@ -872,3 +872,12 @@ its result is kern_unmount() or kern_unmount_array(). zero-length bvec segments are disallowed, they must be filtered out before passed on to an iterator. + +--- + +**mandatory** + +For bvec based itererators bio_iov_iter_get_pages() now doesn't copy bvecs but +uses the one provided. Anyone issuing kiocb-I/O should ensure that the bvec and +page references stay until I/O has completed, i.e. until ->ki_complete() has +been called or returned with non -EIOCBQUEUED code. diff --git a/block/bio.c b/block/bio.c index 1cd8a2e79048..99040a7e6656 100644 --- a/block/bio.c +++ b/block/bio.c @@ -942,21 +942,17 @@ void bio_release_pages(struct bio *bio, bool mark_dirty) } EXPORT_SYMBOL_GPL(bio_release_pages); -static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter) +static int bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter) { - const struct bio_vec *bv = iter->bvec; - unsigned int len; - size_t size; - - if (WARN_ON_ONCE(iter->iov_offset > bv->bv_len)) - return -EINVAL; - - len = min_t(size_t, bv->bv_len - iter->iov_offset, iter->count); - size = bio_add_page(bio, bv->bv_page, len, - bv->bv_offset + iter->iov_offset); - if (unlikely(size != len)) - return -EINVAL; - iov_iter_advance(iter, size); + WARN_ON_ONCE(BVEC_POOL_IDX(bio) != 0); + + bio->bi_vcnt = iter->nr_segs; + bio->bi_max_vecs = iter->nr_segs; + bio->bi_io_vec = (struct bio_vec *)iter->bvec; + bio->bi_iter.bi_bvec_done = iter->iov_offset; + bio->bi_iter.bi_size = iter->count; + + iov_iter_advance(iter, iter->count); return 0; } @@ -1070,12 +1066,12 @@ static int __bio_iov_append_get_pages(struct bio *bio, struct iov_iter *iter) * This takes either an iterator pointing to user memory, or one pointing to * kernel pages (BVEC iterator). If we're adding user pages, we pin them and * map them into the kernel. On IO completion, the caller should put those - * pages. If we're adding kernel pages, and the caller told us it's safe to - * do so, we just have to add the pages to the bio directly. We don't grab an - * extra reference to those pages (the user should already have that), and we - * don't put the page on IO completion. The caller needs to check if the bio is - * flagged BIO_NO_PAGE_REF on IO completion. If it isn't, then pages should be - * released. + * pages. For bvec based iterators bio_iov_iter_get_pages() uses the provided + * bvecs rather than copying them. Hence anyone issuing kiocb based IO needs + * to ensure the bvecs and pages stay referenced until the submitted I/O is + * completed by a call to ->ki_complete() or returns with an error other than + * -EIOCBQUEUED. The caller needs to check if the bio is flagged BIO_NO_PAGE_REF + * on IO completion. If it isn't, then pages should be released. * * The function tries, but does not guarantee, to pin as many pages as * fit into the bio, or are requested in @iter, whatever is smaller. If @@ -1087,27 +1083,22 @@ static int __bio_iov_append_get_pages(struct bio *bio, struct iov_iter *iter) */ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) { - const bool is_bvec = iov_iter_is_bvec(iter); - int ret; - - if (WARN_ON_ONCE(bio->bi_vcnt)) - return -EINVAL; + int ret = 0; - do { - if (bio_op(bio) == REQ_OP_ZONE_APPEND) { - if (WARN_ON_ONCE(is_bvec)) - return -EINVAL; - ret = __bio_iov_append_get_pages(bio, iter); - } else { - if (is_bvec) - ret = __bio_iov_bvec_add_pages(bio, iter); + if (iov_iter_is_bvec(iter)) { + if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND)) + return -EINVAL; + bio_iov_bvec_set(bio, iter); + bio_set_flag(bio, BIO_NO_PAGE_REF); + return 0; + } else { + do { + if (bio_op(bio) == REQ_OP_ZONE_APPEND) + ret = __bio_iov_append_get_pages(bio, iter); else ret = __bio_iov_iter_get_pages(bio, iter); - } - } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0)); - - if (is_bvec) - bio_set_flag(bio, BIO_NO_PAGE_REF); + } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0)); + } /* don't account direct I/O as memory stall */ bio_clear_flag(bio, BIO_WORKINGSET); diff --git a/include/linux/bio.h b/include/linux/bio.h index 9ddb19801a03..676870b2c88d 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -444,10 +444,13 @@ static inline void bio_wouldblock_error(struct bio *bio) /* * Calculate number of bvec segments that should be allocated to fit data - * pointed by @iter. + * pointed by @iter. If @iter is backed by bvec it's going to be reused + * instead of allocating a new one. */ static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs) { + if (iov_iter_is_bvec(iter)) + return 0; return iov_iter_npages(iter, max_segs); } -- cgit v1.2.3 From 9d56653d14cd5e545599cd9e3013daa17df50cd4 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Mon, 25 Jan 2021 12:59:56 +0100 Subject: ACPI: platform-profile: Drop const qualifier for cur_profile Drop the const qualifier from the static global cur_profile pointer declaration. This is a preparation patch for passing the cur_profile pointer as parameter to the profile_get() and profile_set() callbacks so that drivers dynamically allocating their driver-data struct, with their platform_profile_handler struct embedded, can use this pointer to get to their driver-data. Note this also requires dropping the const from the pprof platform_profile_register() function argument. Dropping this const is not a problem, non of the queued up consumers of platform_profile_register() actually pass in a const pointer. Link: https://lore.kernel.org/linux-acpi/5e7a4d87-52ef-e487-9cc2-8e7094beaa08@redhat.com/ Link: https://lore.kernel.org/r/20210114073429.176462-2-jiaxun.yang@flygoat.com Suggested-by: Hans de Goede Signed-off-by: Jiaxun Yang [ hdegoede@redhat.com: Also remove const from platform_profile_register() ] Signed-off-by: Hans de Goede Signed-off-by: Rafael J. Wysocki --- drivers/acpi/platform_profile.c | 4 ++-- include/linux/platform_profile.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/platform_profile.c b/drivers/acpi/platform_profile.c index 91be50a32cc8..f65c61db7921 100644 --- a/drivers/acpi/platform_profile.c +++ b/drivers/acpi/platform_profile.c @@ -9,7 +9,7 @@ #include #include -static const struct platform_profile_handler *cur_profile; +static struct platform_profile_handler *cur_profile; static DEFINE_MUTEX(profile_lock); static const char * const profile_names[] = { @@ -132,7 +132,7 @@ void platform_profile_notify(void) } EXPORT_SYMBOL_GPL(platform_profile_notify); -int platform_profile_register(const struct platform_profile_handler *pprof) +int platform_profile_register(struct platform_profile_handler *pprof) { int err; diff --git a/include/linux/platform_profile.h b/include/linux/platform_profile.h index 3623d7108421..c797fdb3d91a 100644 --- a/include/linux/platform_profile.h +++ b/include/linux/platform_profile.h @@ -32,7 +32,7 @@ struct platform_profile_handler { int (*profile_set)(enum platform_profile_option profile); }; -int platform_profile_register(const struct platform_profile_handler *pprof); +int platform_profile_register(struct platform_profile_handler *pprof); int platform_profile_remove(void); void platform_profile_notify(void); -- cgit v1.2.3 From d17655759b3fc660ea49bd7be665c193030c77c0 Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Sat, 23 Jan 2021 19:34:45 +0300 Subject: reset: Add devm_reset_control_get_optional_exclusive_released() NVIDIA Tegra DRM and media drivers will need a resource-managed-optional variant of reset_control_get_exclusive_released() in order to switch away from a legacy Tegra-specific PD API to a GENPD API without much hassle. Add the new reset helper to the reset API. Tested-by: Peter Geis # Ouya T30 Tested-by: Nicolas Chauvet # PAZ00 T20 Tested-by: Matt Merhar # Ouya T30 Signed-off-by: Dmitry Osipenko Signed-off-by: Philipp Zabel --- include/linux/reset.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'include/linux') diff --git a/include/linux/reset.h b/include/linux/reset.h index 439fec7112a9..b9109efa2a5c 100644 --- a/include/linux/reset.h +++ b/include/linux/reset.h @@ -362,6 +362,25 @@ __must_check devm_reset_control_get_exclusive_released(struct device *dev, return __devm_reset_control_get(dev, id, 0, false, false, false); } +/** + * devm_reset_control_get_optional_exclusive_released - resource managed + * reset_control_get_optional_exclusive_released() + * @dev: device to be reset by the controller + * @id: reset line name + * + * Managed-and-optional variant of reset_control_get_exclusive_released(). For + * reset controllers returned from this function, reset_control_put() is called + * automatically on driver detach. + * + * See reset_control_get_exclusive_released() for more information. + */ +static inline struct reset_control * +__must_check devm_reset_control_get_optional_exclusive_released(struct device *dev, + const char *id) +{ + return __devm_reset_control_get(dev, id, 0, false, true, false); +} + /** * devm_reset_control_get_shared - resource managed reset_control_get_shared() * @dev: device to be reset by the controller -- cgit v1.2.3 From 83ace77f51175023c3757e2d08a92565f9b1c7f3 Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Wed, 20 Jan 2021 16:30:03 +0100 Subject: netfilter: ctnetlink: remove get_ct indirection Use nf_ct_get() directly, its a small inline helper without dependencies. Add CONFIG_NF_CONNTRACK guards to elide the relevant part when conntrack isn't available at all. v2: add ifdef guard around nf_ct_get call (kernel test robot) Signed-off-by: Florian Westphal Signed-off-by: Pablo Neira Ayuso --- include/linux/netfilter.h | 2 -- net/netfilter/nf_conntrack_netlink.c | 7 ------- net/netfilter/nfnetlink_log.c | 8 +++++++- net/netfilter/nfnetlink_queue.c | 10 ++++++++-- 4 files changed, 15 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h index 0101747de549..f0f3a8354c3c 100644 --- a/include/linux/netfilter.h +++ b/include/linux/netfilter.h @@ -463,8 +463,6 @@ extern struct nf_ct_hook __rcu *nf_ct_hook; struct nlattr; struct nfnl_ct_hook { - struct nf_conn *(*get_ct)(const struct sk_buff *skb, - enum ip_conntrack_info *ctinfo); size_t (*build_size)(const struct nf_conn *ct); int (*build)(struct sk_buff *skb, struct nf_conn *ct, enum ip_conntrack_info ctinfo, diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index 84caf3316946..1469365bac7e 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -2686,12 +2686,6 @@ ctnetlink_glue_build_size(const struct nf_conn *ct) ; } -static struct nf_conn *ctnetlink_glue_get_ct(const struct sk_buff *skb, - enum ip_conntrack_info *ctinfo) -{ - return nf_ct_get(skb, ctinfo); -} - static int __ctnetlink_glue_build(struct sk_buff *skb, struct nf_conn *ct) { const struct nf_conntrack_zone *zone; @@ -2925,7 +2919,6 @@ static void ctnetlink_glue_seqadj(struct sk_buff *skb, struct nf_conn *ct, } static struct nfnl_ct_hook ctnetlink_glue_hook = { - .get_ct = ctnetlink_glue_get_ct, .build_size = ctnetlink_glue_build_size, .build = ctnetlink_glue_build, .parse = ctnetlink_glue_parse, diff --git a/net/netfilter/nfnetlink_log.c b/net/netfilter/nfnetlink_log.c index b35e8d9a5b37..26776b88a539 100644 --- a/net/netfilter/nfnetlink_log.c +++ b/net/netfilter/nfnetlink_log.c @@ -43,6 +43,10 @@ #include "../bridge/br_private.h" #endif +#if IS_ENABLED(CONFIG_NF_CONNTRACK) +#include +#endif + #define NFULNL_COPY_DISABLED 0xff #define NFULNL_NLBUFSIZ_DEFAULT NLMSG_GOODSIZE #define NFULNL_TIMEOUT_DEFAULT 100 /* every second */ @@ -733,14 +737,16 @@ nfulnl_log_packet(struct net *net, size += nla_total_size(sizeof(u_int32_t)); if (inst->flags & NFULNL_CFG_F_SEQ_GLOBAL) size += nla_total_size(sizeof(u_int32_t)); +#if IS_ENABLED(CONFIG_NF_CONNTRACK) if (inst->flags & NFULNL_CFG_F_CONNTRACK) { nfnl_ct = rcu_dereference(nfnl_ct_hook); if (nfnl_ct != NULL) { - ct = nfnl_ct->get_ct(skb, &ctinfo); + ct = nf_ct_get(skb, &ctinfo); if (ct != NULL) size += nfnl_ct->build_size(ct); } } +#endif if (pf == NFPROTO_NETDEV || pf == NFPROTO_BRIDGE) size += nfulnl_get_bridge_size(skb); diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index d1d8bca03b4f..48a07914fd94 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -444,13 +444,15 @@ nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue, nfnl_ct = rcu_dereference(nfnl_ct_hook); +#if IS_ENABLED(CONFIG_NF_CONNTRACK) if (queue->flags & NFQA_CFG_F_CONNTRACK) { if (nfnl_ct != NULL) { - ct = nfnl_ct->get_ct(entskb, &ctinfo); + ct = nf_ct_get(entskb, &ctinfo); if (ct != NULL) size += nfnl_ct->build_size(ct); } } +#endif if (queue->flags & NFQA_CFG_F_UID_GID) { size += (nla_total_size(sizeof(u_int32_t)) /* uid */ @@ -1104,9 +1106,10 @@ static struct nf_conn *nfqnl_ct_parse(struct nfnl_ct_hook *nfnl_ct, struct nf_queue_entry *entry, enum ip_conntrack_info *ctinfo) { +#if IS_ENABLED(CONFIG_NF_CONNTRACK) struct nf_conn *ct; - ct = nfnl_ct->get_ct(entry->skb, ctinfo); + ct = nf_ct_get(entry->skb, ctinfo); if (ct == NULL) return NULL; @@ -1118,6 +1121,9 @@ static struct nf_conn *nfqnl_ct_parse(struct nfnl_ct_hook *nfnl_ct, NETLINK_CB(entry->skb).portid, nlmsg_report(nlh)); return ct; +#else + return NULL; +#endif } static int nfqa_parse_bridge(struct nf_queue_entry *entry, -- cgit v1.2.3 From fd765da06066215b5e75c5c2a0db1543a81aa14d Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 21 Jan 2021 08:18:12 +0100 Subject: staging: hikey9xx: hi6421v600-regulator: fix delay logic The original driver, which can be seen at commit 42f24d9d446a ("staging: regulator: add a regulator driver for HiSilicon 6421v600 SPMI PMIC") had a complex logic to ensure that there won't be multiple power enable/disable commands running at the same time. At the original logic, it were ensured that: - a next power up/down would wait for at least the on/off period; - an extra delay would be granted. It turns that such extra delay has a value of zero, but it was relying on gettimeofday() call, which can take some time. This was later simplified, but there are still some possible issues. In order to avoid that, let's simply add a delay to wait for the power up line to stabilize after powering up a device. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/6733dac9813ba6688def404142cb7b964accf758.1611212783.git.mchehab+huawei@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/staging/hikey9xx/hi6421v600-regulator.c | 5 +++-- include/linux/mfd/hi6421-spmi-pmic.h | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/staging/hikey9xx/hi6421v600-regulator.c b/drivers/staging/hikey9xx/hi6421v600-regulator.c index 5e78eebfc1f3..e5a492ee7121 100644 --- a/drivers/staging/hikey9xx/hi6421v600-regulator.c +++ b/drivers/staging/hikey9xx/hi6421v600-regulator.c @@ -113,14 +113,15 @@ static int hi6421_spmi_regulator_enable(struct regulator_dev *rdev) /* cannot enable more than one regulator at one time */ mutex_lock(&sreg->enable_mutex); - usleep_range(HISI_REGS_ENA_PROTECT_TIME, - HISI_REGS_ENA_PROTECT_TIME + 1000); /* set enable register */ ret = hi6421_spmi_pmic_rmw(pmic, rdev->desc->enable_reg, rdev->desc->enable_mask, rdev->desc->enable_mask); + /* Avoid powering up multiple devices at the same time */ + usleep_range(rdev->desc->off_on_delay, rdev->desc->off_on_delay + 60); + mutex_unlock(&sreg->enable_mutex); return ret; diff --git a/include/linux/mfd/hi6421-spmi-pmic.h b/include/linux/mfd/hi6421-spmi-pmic.h index 2c8896fd852e..0c2214612c4e 100644 --- a/include/linux/mfd/hi6421-spmi-pmic.h +++ b/include/linux/mfd/hi6421-spmi-pmic.h @@ -13,7 +13,6 @@ #include -#define HISI_REGS_ENA_PROTECT_TIME (0) /* in microseconds */ #define HISI_ECO_MODE_ENABLE (1) #define HISI_ECO_MODE_DISABLE (0) -- cgit v1.2.3 From e9103f47bf1a1bbf0ab0ea90eda3e208653a5f57 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 25 Jan 2021 17:02:38 +0200 Subject: serial: ifx6x60: Remove driver for deprecated platform Intel Moorestown and Medfield are quite old Intel Atom based 32-bit platforms, which were in limited use in some Android phones, tablets and consumer electronics more than eight years ago. There are no bugs or problems ever reported outside from Intel for breaking any of that platforms for years. It seems no real users exists who run more or less fresh kernel on it. The commit 05f4434bc130 ("ASoC: Intel: remove mfld_machine") also in align with this theory. Due to above and to reduce a burden of supporting outdated drivers we remove the support of outdated platforms completely. Signed-off-by: Andy Shevchenko Acked-by: Linus Walleij Link: https://lore.kernel.org/r/20210125150238.16980-1-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/Kconfig | 7 - drivers/tty/serial/Makefile | 1 - drivers/tty/serial/ifx6x60.c | 1387 ----------------------------------------- drivers/tty/serial/ifx6x60.h | 118 ---- include/linux/spi/ifx_modem.h | 15 - 5 files changed, 1528 deletions(-) delete mode 100644 drivers/tty/serial/ifx6x60.c delete mode 100644 drivers/tty/serial/ifx6x60.h delete mode 100644 include/linux/spi/ifx_modem.h (limited to 'include/linux') diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig index 22e86ba4f827..0c4cd4a348f4 100644 --- a/drivers/tty/serial/Kconfig +++ b/drivers/tty/serial/Kconfig @@ -1182,13 +1182,6 @@ config SERIAL_ALTERA_UART_CONSOLE help Enable a Altera UART port to be the system console. -config SERIAL_IFX6X60 - tristate "SPI protocol driver for Infineon 6x60 modem (EXPERIMENTAL)" - depends on GPIOLIB || COMPILE_TEST - depends on SPI && HAS_DMA - help - Support for the IFX6x60 modem devices on Intel MID platforms. - config SERIAL_PCH_UART tristate "Intel EG20T PCH/LAPIS Semicon IOH(ML7213/ML7223/ML7831) UART" depends on PCI && (X86_32 || MIPS || COMPILE_TEST) diff --git a/drivers/tty/serial/Makefile b/drivers/tty/serial/Makefile index 931dc1d6885e..7da0856cd198 100644 --- a/drivers/tty/serial/Makefile +++ b/drivers/tty/serial/Makefile @@ -64,7 +64,6 @@ obj-$(CONFIG_SERIAL_TIMBERDALE) += timbuart.o obj-$(CONFIG_SERIAL_GRLIB_GAISLER_APBUART) += apbuart.o obj-$(CONFIG_SERIAL_ALTERA_JTAGUART) += altera_jtaguart.o obj-$(CONFIG_SERIAL_VT8500) += vt8500_serial.o -obj-$(CONFIG_SERIAL_IFX6X60) += ifx6x60.o obj-$(CONFIG_SERIAL_PCH_UART) += pch_uart.o obj-$(CONFIG_SERIAL_MXS_AUART) += mxs-auart.o obj-$(CONFIG_SERIAL_LANTIQ) += lantiq.o diff --git a/drivers/tty/serial/ifx6x60.c b/drivers/tty/serial/ifx6x60.c deleted file mode 100644 index d4ef88ee22d0..000000000000 --- a/drivers/tty/serial/ifx6x60.c +++ /dev/null @@ -1,1387 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/**************************************************************************** - * - * Driver for the IFX 6x60 spi modem. - * - * Copyright (C) 2008 Option International - * Copyright (C) 2008 Filip Aben - * Denis Joseph Barrow - * Jan Dumon - * - * Copyright (C) 2009, 2010 Intel Corp - * Russ Gorby - * - * Driver modified by Intel from Option gtm501l_spi.c - * - * Notes - * o The driver currently assumes a single device only. If you need to - * change this then look for saved_ifx_dev and add a device lookup - * o The driver is intended to be big-endian safe but has never been - * tested that way (no suitable hardware). There are a couple of FIXME - * notes by areas that may need addressing - * o Some of the GPIO naming/setup assumptions may need revisiting if - * you need to use this driver for another platform. - * - *****************************************************************************/ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ifx6x60.h" - -#define IFX_SPI_MORE_MASK 0x10 -#define IFX_SPI_MORE_BIT 4 /* bit position in u8 */ -#define IFX_SPI_CTS_BIT 6 /* bit position in u8 */ -#define IFX_SPI_MODE SPI_MODE_1 -#define IFX_SPI_TTY_ID 0 -#define IFX_SPI_TIMEOUT_SEC 2 -#define IFX_SPI_HEADER_0 (-1) -#define IFX_SPI_HEADER_F (-2) - -#define PO_POST_DELAY 200 - -/* forward reference */ -static void ifx_spi_handle_srdy(struct ifx_spi_device *ifx_dev); -static int ifx_modem_reboot_callback(struct notifier_block *nfb, - unsigned long event, void *data); -static int ifx_modem_power_off(struct ifx_spi_device *ifx_dev); - -/* local variables */ -static int spi_bpw = 16; /* 8, 16 or 32 bit word length */ -static struct tty_driver *tty_drv; -static struct ifx_spi_device *saved_ifx_dev; -static struct lock_class_key ifx_spi_key; - -static struct notifier_block ifx_modem_reboot_notifier_block = { - .notifier_call = ifx_modem_reboot_callback, -}; - -static int ifx_modem_power_off(struct ifx_spi_device *ifx_dev) -{ - gpiod_set_value(ifx_dev->gpio.pmu_reset, 1); - msleep(PO_POST_DELAY); - - return 0; -} - -static int ifx_modem_reboot_callback(struct notifier_block *nfb, - unsigned long event, void *data) -{ - if (saved_ifx_dev) - ifx_modem_power_off(saved_ifx_dev); - else - pr_warn("no ifx modem active;\n"); - - return NOTIFY_OK; -} - -/* GPIO/GPE settings */ - -/** - * mrdy_set_high - set MRDY GPIO - * @ifx: device we are controlling - * - */ -static inline void mrdy_set_high(struct ifx_spi_device *ifx) -{ - gpiod_set_value(ifx->gpio.mrdy, 1); -} - -/** - * mrdy_set_low - clear MRDY GPIO - * @ifx: device we are controlling - * - */ -static inline void mrdy_set_low(struct ifx_spi_device *ifx) -{ - gpiod_set_value(ifx->gpio.mrdy, 0); -} - -/** - * ifx_spi_power_state_set - * @ifx_dev: our SPI device - * @val: bits to set - * - * Set bit in power status and signal power system if status becomes non-0 - */ -static void -ifx_spi_power_state_set(struct ifx_spi_device *ifx_dev, unsigned char val) -{ - unsigned long flags; - - spin_lock_irqsave(&ifx_dev->power_lock, flags); - - /* - * if power status is already non-0, just update, else - * tell power system - */ - if (!ifx_dev->power_status) - pm_runtime_get(&ifx_dev->spi_dev->dev); - ifx_dev->power_status |= val; - - spin_unlock_irqrestore(&ifx_dev->power_lock, flags); -} - -/** - * ifx_spi_power_state_clear - clear power bit - * @ifx_dev: our SPI device - * @val: bits to clear - * - * clear bit in power status and signal power system if status becomes 0 - */ -static void -ifx_spi_power_state_clear(struct ifx_spi_device *ifx_dev, unsigned char val) -{ - unsigned long flags; - - spin_lock_irqsave(&ifx_dev->power_lock, flags); - - if (ifx_dev->power_status) { - ifx_dev->power_status &= ~val; - if (!ifx_dev->power_status) - pm_runtime_put(&ifx_dev->spi_dev->dev); - } - - spin_unlock_irqrestore(&ifx_dev->power_lock, flags); -} - -/** - * swap_buf_8 - * @buf: our buffer - * @len : number of bytes (not words) in the buffer - * @end: end of buffer - * - * Swap the contents of a buffer into big endian format - */ -static inline void swap_buf_8(unsigned char *buf, int len, void *end) -{ - /* don't swap buffer if SPI word width is 8 bits */ - return; -} - -/** - * swap_buf_16 - * @buf: our buffer - * @len : number of bytes (not words) in the buffer - * @end: end of buffer - * - * Swap the contents of a buffer into big endian format - */ -static inline void swap_buf_16(unsigned char *buf, int len, void *end) -{ - int n; - - u16 *buf_16 = (u16 *)buf; - len = ((len + 1) >> 1); - if ((void *)&buf_16[len] > end) { - pr_err("swap_buf_16: swap exceeds boundary (%p > %p)!", - &buf_16[len], end); - return; - } - for (n = 0; n < len; n++) { - *buf_16 = cpu_to_be16(*buf_16); - buf_16++; - } -} - -/** - * swap_buf_32 - * @buf: our buffer - * @len : number of bytes (not words) in the buffer - * @end: end of buffer - * - * Swap the contents of a buffer into big endian format - */ -static inline void swap_buf_32(unsigned char *buf, int len, void *end) -{ - int n; - - u32 *buf_32 = (u32 *)buf; - len = (len + 3) >> 2; - - if ((void *)&buf_32[len] > end) { - pr_err("swap_buf_32: swap exceeds boundary (%p > %p)!\n", - &buf_32[len], end); - return; - } - for (n = 0; n < len; n++) { - *buf_32 = cpu_to_be32(*buf_32); - buf_32++; - } -} - -/** - * mrdy_assert - assert MRDY line - * @ifx_dev: our SPI device - * - * Assert mrdy and set timer to wait for SRDY interrupt, if SRDY is low - * now. - * - * FIXME: Can SRDY even go high as we are running this code ? - */ -static void mrdy_assert(struct ifx_spi_device *ifx_dev) -{ - int val = gpiod_get_value(ifx_dev->gpio.srdy); - if (!val) { - if (!test_and_set_bit(IFX_SPI_STATE_TIMER_PENDING, - &ifx_dev->flags)) { - mod_timer(&ifx_dev->spi_timer,jiffies + IFX_SPI_TIMEOUT_SEC*HZ); - - } - } - ifx_spi_power_state_set(ifx_dev, IFX_SPI_POWER_DATA_PENDING); - mrdy_set_high(ifx_dev); -} - -/** - * ifx_spi_timeout - SPI timeout - * @t: timer in our SPI device - * - * The SPI has timed out: hang up the tty. Users will then see a hangup - * and error events. - */ -static void ifx_spi_timeout(struct timer_list *t) -{ - struct ifx_spi_device *ifx_dev = from_timer(ifx_dev, t, spi_timer); - - dev_warn(&ifx_dev->spi_dev->dev, "*** SPI Timeout ***"); - tty_port_tty_hangup(&ifx_dev->tty_port, false); - mrdy_set_low(ifx_dev); - clear_bit(IFX_SPI_STATE_TIMER_PENDING, &ifx_dev->flags); -} - -/* char/tty operations */ - -/** - * ifx_spi_tiocmget - get modem lines - * @tty: our tty device - * - * Map the signal state into Linux modem flags and report the value - * in Linux terms - */ -static int ifx_spi_tiocmget(struct tty_struct *tty) -{ - unsigned int value; - struct ifx_spi_device *ifx_dev = tty->driver_data; - - value = - (test_bit(IFX_SPI_RTS, &ifx_dev->signal_state) ? TIOCM_RTS : 0) | - (test_bit(IFX_SPI_DTR, &ifx_dev->signal_state) ? TIOCM_DTR : 0) | - (test_bit(IFX_SPI_CTS, &ifx_dev->signal_state) ? TIOCM_CTS : 0) | - (test_bit(IFX_SPI_DSR, &ifx_dev->signal_state) ? TIOCM_DSR : 0) | - (test_bit(IFX_SPI_DCD, &ifx_dev->signal_state) ? TIOCM_CAR : 0) | - (test_bit(IFX_SPI_RI, &ifx_dev->signal_state) ? TIOCM_RNG : 0); - return value; -} - -/** - * ifx_spi_tiocmset - set modem bits - * @tty: the tty structure - * @set: bits to set - * @clear: bits to clear - * - * The IFX6x60 only supports DTR and RTS. Set them accordingly - * and flag that an update to the modem is needed. - * - * FIXME: do we need to kick the tranfers when we do this ? - */ -static int ifx_spi_tiocmset(struct tty_struct *tty, - unsigned int set, unsigned int clear) -{ - struct ifx_spi_device *ifx_dev = tty->driver_data; - - if (set & TIOCM_RTS) - set_bit(IFX_SPI_RTS, &ifx_dev->signal_state); - if (set & TIOCM_DTR) - set_bit(IFX_SPI_DTR, &ifx_dev->signal_state); - if (clear & TIOCM_RTS) - clear_bit(IFX_SPI_RTS, &ifx_dev->signal_state); - if (clear & TIOCM_DTR) - clear_bit(IFX_SPI_DTR, &ifx_dev->signal_state); - - set_bit(IFX_SPI_UPDATE, &ifx_dev->signal_state); - return 0; -} - -/** - * ifx_spi_open - called on tty open - * @tty: our tty device - * @filp: file handle being associated with the tty - * - * Open the tty interface. We let the tty_port layer do all the work - * for us. - * - * FIXME: Remove single device assumption and saved_ifx_dev - */ -static int ifx_spi_open(struct tty_struct *tty, struct file *filp) -{ - return tty_port_open(&saved_ifx_dev->tty_port, tty, filp); -} - -/** - * ifx_spi_close - called when our tty closes - * @tty: the tty being closed - * @filp: the file handle being closed - * - * Perform the close of the tty. We use the tty_port layer to do all - * our hard work. - */ -static void ifx_spi_close(struct tty_struct *tty, struct file *filp) -{ - struct ifx_spi_device *ifx_dev = tty->driver_data; - tty_port_close(&ifx_dev->tty_port, tty, filp); - /* FIXME: should we do an ifx_spi_reset here ? */ -} - -/** - * ifx_decode_spi_header - decode received header - * @buffer: the received data - * @length: decoded length - * @more: decoded more flag - * @received_cts: status of cts we received - * - * Note how received_cts is handled -- if header is all F it is left - * the same as it was, if header is all 0 it is set to 0 otherwise it is - * taken from the incoming header. - * - * FIXME: endianness - */ -static int ifx_spi_decode_spi_header(unsigned char *buffer, int *length, - unsigned char *more, unsigned char *received_cts) -{ - u16 h1; - u16 h2; - u16 *in_buffer = (u16 *)buffer; - - h1 = *in_buffer; - h2 = *(in_buffer+1); - - if (h1 == 0 && h2 == 0) { - *received_cts = 0; - *more = 0; - return IFX_SPI_HEADER_0; - } else if (h1 == 0xffff && h2 == 0xffff) { - *more = 0; - /* spi_slave_cts remains as it was */ - return IFX_SPI_HEADER_F; - } - - *length = h1 & 0xfff; /* upper bits of byte are flags */ - *more = (buffer[1] >> IFX_SPI_MORE_BIT) & 1; - *received_cts = (buffer[3] >> IFX_SPI_CTS_BIT) & 1; - return 0; -} - -/** - * ifx_setup_spi_header - set header fields - * @txbuffer: pointer to start of SPI buffer - * @tx_count: bytes - * @more: indicate if more to follow - * - * Format up an SPI header for a transfer - * - * FIXME: endianness? - */ -static void ifx_spi_setup_spi_header(unsigned char *txbuffer, int tx_count, - unsigned char more) -{ - *(u16 *)(txbuffer) = tx_count; - *(u16 *)(txbuffer+2) = IFX_SPI_PAYLOAD_SIZE; - txbuffer[1] |= (more << IFX_SPI_MORE_BIT) & IFX_SPI_MORE_MASK; -} - -/** - * ifx_spi_prepare_tx_buffer - prepare transmit frame - * @ifx_dev: our SPI device - * - * The transmit buffr needs a header and various other bits of - * information followed by as much data as we can pull from the FIFO - * and transfer. This function formats up a suitable buffer in the - * ifx_dev->tx_buffer - * - * FIXME: performance - should we wake the tty when the queue is half - * empty ? - */ -static int ifx_spi_prepare_tx_buffer(struct ifx_spi_device *ifx_dev) -{ - int temp_count; - int queue_length; - int tx_count; - unsigned char *tx_buffer; - - tx_buffer = ifx_dev->tx_buffer; - - /* make room for required SPI header */ - tx_buffer += IFX_SPI_HEADER_OVERHEAD; - tx_count = IFX_SPI_HEADER_OVERHEAD; - - /* clear to signal no more data if this turns out to be the - * last buffer sent in a sequence */ - ifx_dev->spi_more = 0; - - /* if modem cts is set, just send empty buffer */ - if (!ifx_dev->spi_slave_cts) { - /* see if there's tx data */ - queue_length = kfifo_len(&ifx_dev->tx_fifo); - if (queue_length != 0) { - /* data to mux -- see if there's room for it */ - temp_count = min(queue_length, IFX_SPI_PAYLOAD_SIZE); - temp_count = kfifo_out_locked(&ifx_dev->tx_fifo, - tx_buffer, temp_count, - &ifx_dev->fifo_lock); - - /* update buffer pointer and data count in message */ - tx_buffer += temp_count; - tx_count += temp_count; - if (temp_count == queue_length) - /* poke port to get more data */ - tty_port_tty_wakeup(&ifx_dev->tty_port); - else /* more data in port, use next SPI message */ - ifx_dev->spi_more = 1; - } - } - /* have data and info for header -- set up SPI header in buffer */ - /* spi header needs payload size, not entire buffer size */ - ifx_spi_setup_spi_header(ifx_dev->tx_buffer, - tx_count-IFX_SPI_HEADER_OVERHEAD, - ifx_dev->spi_more); - /* swap actual data in the buffer */ - ifx_dev->swap_buf((ifx_dev->tx_buffer), tx_count, - &ifx_dev->tx_buffer[IFX_SPI_TRANSFER_SIZE]); - return tx_count; -} - -/** - * ifx_spi_write - line discipline write - * @tty: our tty device - * @buf: pointer to buffer to write (kernel space) - * @count: size of buffer - * - * Write the characters we have been given into the FIFO. If the device - * is not active then activate it, when the SRDY line is asserted back - * this will commence I/O - */ -static int ifx_spi_write(struct tty_struct *tty, const unsigned char *buf, - int count) -{ - struct ifx_spi_device *ifx_dev = tty->driver_data; - unsigned char *tmp_buf = (unsigned char *)buf; - unsigned long flags; - bool is_fifo_empty; - int tx_count; - - spin_lock_irqsave(&ifx_dev->fifo_lock, flags); - is_fifo_empty = kfifo_is_empty(&ifx_dev->tx_fifo); - tx_count = kfifo_in(&ifx_dev->tx_fifo, tmp_buf, count); - spin_unlock_irqrestore(&ifx_dev->fifo_lock, flags); - if (is_fifo_empty) - mrdy_assert(ifx_dev); - - return tx_count; -} - -/** - * ifx_spi_chars_in_buffer - line discipline helper - * @tty: our tty device - * - * Report how much data we can accept before we drop bytes. As we use - * a simple FIFO this is nice and easy. - */ -static int ifx_spi_write_room(struct tty_struct *tty) -{ - struct ifx_spi_device *ifx_dev = tty->driver_data; - return IFX_SPI_FIFO_SIZE - kfifo_len(&ifx_dev->tx_fifo); -} - -/** - * ifx_spi_chars_in_buffer - line discipline helper - * @tty: our tty device - * - * Report how many characters we have buffered. In our case this is the - * number of bytes sitting in our transmit FIFO. - */ -static int ifx_spi_chars_in_buffer(struct tty_struct *tty) -{ - struct ifx_spi_device *ifx_dev = tty->driver_data; - return kfifo_len(&ifx_dev->tx_fifo); -} - -/** - * ifx_port_hangup - * @tty: our tty - * - * tty port hang up. Called when tty_hangup processing is invoked either - * by loss of carrier, or by software (eg vhangup). Serialized against - * activate/shutdown by the tty layer. - */ -static void ifx_spi_hangup(struct tty_struct *tty) -{ - struct ifx_spi_device *ifx_dev = tty->driver_data; - tty_port_hangup(&ifx_dev->tty_port); -} - -/** - * ifx_port_activate - * @port: our tty port - * @tty: our tty device - * - * tty port activate method - called for first open. Serialized - * with hangup and shutdown by the tty layer. - */ -static int ifx_port_activate(struct tty_port *port, struct tty_struct *tty) -{ - struct ifx_spi_device *ifx_dev = - container_of(port, struct ifx_spi_device, tty_port); - - /* clear any old data; can't do this in 'close' */ - kfifo_reset(&ifx_dev->tx_fifo); - - /* clear any flag which may be set in port shutdown procedure */ - clear_bit(IFX_SPI_STATE_IO_IN_PROGRESS, &ifx_dev->flags); - clear_bit(IFX_SPI_STATE_IO_READY, &ifx_dev->flags); - - /* put port data into this tty */ - tty->driver_data = ifx_dev; - - /* set flag to allows data transfer */ - set_bit(IFX_SPI_STATE_IO_AVAILABLE, &ifx_dev->flags); - - return 0; -} - -/** - * ifx_port_shutdown - * @port: our tty port - * - * tty port shutdown method - called for last port close. Serialized - * with hangup and activate by the tty layer. - */ -static void ifx_port_shutdown(struct tty_port *port) -{ - struct ifx_spi_device *ifx_dev = - container_of(port, struct ifx_spi_device, tty_port); - - clear_bit(IFX_SPI_STATE_IO_AVAILABLE, &ifx_dev->flags); - mrdy_set_low(ifx_dev); - del_timer(&ifx_dev->spi_timer); - clear_bit(IFX_SPI_STATE_TIMER_PENDING, &ifx_dev->flags); - tasklet_kill(&ifx_dev->io_work_tasklet); -} - -static const struct tty_port_operations ifx_tty_port_ops = { - .activate = ifx_port_activate, - .shutdown = ifx_port_shutdown, -}; - -static const struct tty_operations ifx_spi_serial_ops = { - .open = ifx_spi_open, - .close = ifx_spi_close, - .write = ifx_spi_write, - .hangup = ifx_spi_hangup, - .write_room = ifx_spi_write_room, - .chars_in_buffer = ifx_spi_chars_in_buffer, - .tiocmget = ifx_spi_tiocmget, - .tiocmset = ifx_spi_tiocmset, -}; - -/** - * ifx_spi_insert_fip_string - queue received data - * @ifx_dev: our SPI device - * @chars: buffer we have received - * @size: number of chars reeived - * - * Queue bytes to the tty assuming the tty side is currently open. If - * not the discard the data. - */ -static void ifx_spi_insert_flip_string(struct ifx_spi_device *ifx_dev, - unsigned char *chars, size_t size) -{ - tty_insert_flip_string(&ifx_dev->tty_port, chars, size); - tty_flip_buffer_push(&ifx_dev->tty_port); -} - -/** - * ifx_spi_complete - SPI transfer completed - * @ctx: our SPI device - * - * An SPI transfer has completed. Process any received data and kick off - * any further transmits we can commence. - */ -static void ifx_spi_complete(void *ctx) -{ - struct ifx_spi_device *ifx_dev = ctx; - int length; - int actual_length; - unsigned char more = 0; - unsigned char cts; - int local_write_pending = 0; - int queue_length; - int srdy; - int decode_result; - - mrdy_set_low(ifx_dev); - - if (!ifx_dev->spi_msg.status) { - /* check header validity, get comm flags */ - ifx_dev->swap_buf(ifx_dev->rx_buffer, IFX_SPI_HEADER_OVERHEAD, - &ifx_dev->rx_buffer[IFX_SPI_HEADER_OVERHEAD]); - decode_result = ifx_spi_decode_spi_header(ifx_dev->rx_buffer, - &length, &more, &cts); - if (decode_result == IFX_SPI_HEADER_0) { - dev_dbg(&ifx_dev->spi_dev->dev, - "ignore input: invalid header 0"); - ifx_dev->spi_slave_cts = 0; - goto complete_exit; - } else if (decode_result == IFX_SPI_HEADER_F) { - dev_dbg(&ifx_dev->spi_dev->dev, - "ignore input: invalid header F"); - goto complete_exit; - } - - ifx_dev->spi_slave_cts = cts; - - actual_length = min((unsigned int)length, - ifx_dev->spi_msg.actual_length); - ifx_dev->swap_buf( - (ifx_dev->rx_buffer + IFX_SPI_HEADER_OVERHEAD), - actual_length, - &ifx_dev->rx_buffer[IFX_SPI_TRANSFER_SIZE]); - ifx_spi_insert_flip_string( - ifx_dev, - ifx_dev->rx_buffer + IFX_SPI_HEADER_OVERHEAD, - (size_t)actual_length); - } else { - more = 0; - dev_dbg(&ifx_dev->spi_dev->dev, "SPI transfer error %d", - ifx_dev->spi_msg.status); - } - -complete_exit: - if (ifx_dev->write_pending) { - ifx_dev->write_pending = 0; - local_write_pending = 1; - } - - clear_bit(IFX_SPI_STATE_IO_IN_PROGRESS, &(ifx_dev->flags)); - - queue_length = kfifo_len(&ifx_dev->tx_fifo); - srdy = gpiod_get_value(ifx_dev->gpio.srdy); - if (!srdy) - ifx_spi_power_state_clear(ifx_dev, IFX_SPI_POWER_SRDY); - - /* schedule output if there is more to do */ - if (test_and_clear_bit(IFX_SPI_STATE_IO_READY, &ifx_dev->flags)) - tasklet_schedule(&ifx_dev->io_work_tasklet); - else { - if (more || ifx_dev->spi_more || queue_length > 0 || - local_write_pending) { - if (ifx_dev->spi_slave_cts) { - if (more) - mrdy_assert(ifx_dev); - } else - mrdy_assert(ifx_dev); - } else { - /* - * poke line discipline driver if any for more data - * may or may not get more data to write - * for now, say not busy - */ - ifx_spi_power_state_clear(ifx_dev, - IFX_SPI_POWER_DATA_PENDING); - tty_port_tty_wakeup(&ifx_dev->tty_port); - } - } -} - -/** - * ifx_spio_io - I/O tasklet - * @t: tasklet construct used to fetch the SPI device - * - * Queue data for transmission if possible and then kick off the - * transfer. - */ -static void ifx_spi_io(struct tasklet_struct *t) -{ - int retval; - struct ifx_spi_device *ifx_dev = from_tasklet(ifx_dev, t, - io_work_tasklet); - - if (!test_and_set_bit(IFX_SPI_STATE_IO_IN_PROGRESS, &ifx_dev->flags) && - test_bit(IFX_SPI_STATE_IO_AVAILABLE, &ifx_dev->flags)) { - if (ifx_dev->gpio.unack_srdy_int_nb > 0) - ifx_dev->gpio.unack_srdy_int_nb--; - - ifx_spi_prepare_tx_buffer(ifx_dev); - - spi_message_init(&ifx_dev->spi_msg); - INIT_LIST_HEAD(&ifx_dev->spi_msg.queue); - - ifx_dev->spi_msg.context = ifx_dev; - ifx_dev->spi_msg.complete = ifx_spi_complete; - - /* set up our spi transfer */ - /* note len is BYTES, not transfers */ - ifx_dev->spi_xfer.len = IFX_SPI_TRANSFER_SIZE; - ifx_dev->spi_xfer.cs_change = 0; - ifx_dev->spi_xfer.speed_hz = ifx_dev->spi_dev->max_speed_hz; - /* ifx_dev->spi_xfer.speed_hz = 390625; */ - ifx_dev->spi_xfer.bits_per_word = - ifx_dev->spi_dev->bits_per_word; - - ifx_dev->spi_xfer.tx_buf = ifx_dev->tx_buffer; - ifx_dev->spi_xfer.rx_buf = ifx_dev->rx_buffer; - - /* - * setup dma pointers - */ - if (ifx_dev->use_dma) { - ifx_dev->spi_msg.is_dma_mapped = 1; - ifx_dev->tx_dma = ifx_dev->tx_bus; - ifx_dev->rx_dma = ifx_dev->rx_bus; - ifx_dev->spi_xfer.tx_dma = ifx_dev->tx_dma; - ifx_dev->spi_xfer.rx_dma = ifx_dev->rx_dma; - } else { - ifx_dev->spi_msg.is_dma_mapped = 0; - ifx_dev->tx_dma = (dma_addr_t)0; - ifx_dev->rx_dma = (dma_addr_t)0; - ifx_dev->spi_xfer.tx_dma = (dma_addr_t)0; - ifx_dev->spi_xfer.rx_dma = (dma_addr_t)0; - } - - spi_message_add_tail(&ifx_dev->spi_xfer, &ifx_dev->spi_msg); - - /* Assert MRDY. This may have already been done by the write - * routine. - */ - mrdy_assert(ifx_dev); - - retval = spi_async(ifx_dev->spi_dev, &ifx_dev->spi_msg); - if (retval) { - clear_bit(IFX_SPI_STATE_IO_IN_PROGRESS, - &ifx_dev->flags); - tasklet_schedule(&ifx_dev->io_work_tasklet); - return; - } - } else - ifx_dev->write_pending = 1; -} - -/** - * ifx_spi_free_port - free up the tty side - * @ifx_dev: IFX device going away - * - * Unregister and free up a port when the device goes away - */ -static void ifx_spi_free_port(struct ifx_spi_device *ifx_dev) -{ - if (ifx_dev->tty_dev) - tty_unregister_device(tty_drv, ifx_dev->minor); - tty_port_destroy(&ifx_dev->tty_port); - kfifo_free(&ifx_dev->tx_fifo); -} - -/** - * ifx_spi_create_port - create a new port - * @ifx_dev: our spi device - * - * Allocate and initialise the tty port that goes with this interface - * and add it to the tty layer so that it can be opened. - */ -static int ifx_spi_create_port(struct ifx_spi_device *ifx_dev) -{ - int ret = 0; - struct tty_port *pport = &ifx_dev->tty_port; - - spin_lock_init(&ifx_dev->fifo_lock); - lockdep_set_class_and_subclass(&ifx_dev->fifo_lock, - &ifx_spi_key, 0); - - if (kfifo_alloc(&ifx_dev->tx_fifo, IFX_SPI_FIFO_SIZE, GFP_KERNEL)) { - ret = -ENOMEM; - goto error_ret; - } - - tty_port_init(pport); - pport->ops = &ifx_tty_port_ops; - ifx_dev->minor = IFX_SPI_TTY_ID; - ifx_dev->tty_dev = tty_port_register_device(pport, tty_drv, - ifx_dev->minor, &ifx_dev->spi_dev->dev); - if (IS_ERR(ifx_dev->tty_dev)) { - dev_dbg(&ifx_dev->spi_dev->dev, - "%s: registering tty device failed", __func__); - ret = PTR_ERR(ifx_dev->tty_dev); - goto error_port; - } - return 0; - -error_port: - tty_port_destroy(pport); -error_ret: - ifx_spi_free_port(ifx_dev); - return ret; -} - -/** - * ifx_spi_handle_srdy - handle SRDY - * @ifx_dev: device asserting SRDY - * - * Check our device state and see what we need to kick off when SRDY - * is asserted. This usually means killing the timer and firing off the - * I/O processing. - */ -static void ifx_spi_handle_srdy(struct ifx_spi_device *ifx_dev) -{ - if (test_bit(IFX_SPI_STATE_TIMER_PENDING, &ifx_dev->flags)) { - del_timer(&ifx_dev->spi_timer); - clear_bit(IFX_SPI_STATE_TIMER_PENDING, &ifx_dev->flags); - } - - ifx_spi_power_state_set(ifx_dev, IFX_SPI_POWER_SRDY); - - if (!test_bit(IFX_SPI_STATE_IO_IN_PROGRESS, &ifx_dev->flags)) - tasklet_schedule(&ifx_dev->io_work_tasklet); - else - set_bit(IFX_SPI_STATE_IO_READY, &ifx_dev->flags); -} - -/** - * ifx_spi_srdy_interrupt - SRDY asserted - * @irq: our IRQ number - * @dev: our ifx device - * - * The modem asserted SRDY. Handle the srdy event - */ -static irqreturn_t ifx_spi_srdy_interrupt(int irq, void *dev) -{ - struct ifx_spi_device *ifx_dev = dev; - ifx_dev->gpio.unack_srdy_int_nb++; - ifx_spi_handle_srdy(ifx_dev); - return IRQ_HANDLED; -} - -/** - * ifx_spi_reset_interrupt - Modem has changed reset state - * @irq: interrupt number - * @dev: our device pointer - * - * The modem has either entered or left reset state. Check the GPIO - * line to see which. - * - * FIXME: review locking on MR_INPROGRESS versus - * parallel unsolicited reset/solicited reset - */ -static irqreturn_t ifx_spi_reset_interrupt(int irq, void *dev) -{ - struct ifx_spi_device *ifx_dev = dev; - int val = gpiod_get_value(ifx_dev->gpio.reset_out); - int solreset = test_bit(MR_START, &ifx_dev->mdm_reset_state); - - if (val == 0) { - /* entered reset */ - set_bit(MR_INPROGRESS, &ifx_dev->mdm_reset_state); - if (!solreset) { - /* unsolicited reset */ - tty_port_tty_hangup(&ifx_dev->tty_port, false); - } - } else { - /* exited reset */ - clear_bit(MR_INPROGRESS, &ifx_dev->mdm_reset_state); - if (solreset) { - set_bit(MR_COMPLETE, &ifx_dev->mdm_reset_state); - wake_up(&ifx_dev->mdm_reset_wait); - } - } - return IRQ_HANDLED; -} - -/** - * ifx_spi_free_device - free device - * @ifx_dev: device to free - * - * Free the IFX device - */ -static void ifx_spi_free_device(struct ifx_spi_device *ifx_dev) -{ - ifx_spi_free_port(ifx_dev); - dma_free_coherent(&ifx_dev->spi_dev->dev, - IFX_SPI_TRANSFER_SIZE, - ifx_dev->tx_buffer, - ifx_dev->tx_bus); - dma_free_coherent(&ifx_dev->spi_dev->dev, - IFX_SPI_TRANSFER_SIZE, - ifx_dev->rx_buffer, - ifx_dev->rx_bus); -} - -/** - * ifx_spi_reset - reset modem - * @ifx_dev: modem to reset - * - * Perform a reset on the modem - */ -static int ifx_spi_reset(struct ifx_spi_device *ifx_dev) -{ - int ret; - /* - * set up modem power, reset - * - * delays are required on some platforms for the modem - * to reset properly - */ - set_bit(MR_START, &ifx_dev->mdm_reset_state); - gpiod_set_value(ifx_dev->gpio.po, 0); - gpiod_set_value(ifx_dev->gpio.reset, 0); - msleep(25); - gpiod_set_value(ifx_dev->gpio.reset, 1); - msleep(1); - gpiod_set_value(ifx_dev->gpio.po, 1); - msleep(1); - gpiod_set_value(ifx_dev->gpio.po, 0); - ret = wait_event_timeout(ifx_dev->mdm_reset_wait, - test_bit(MR_COMPLETE, - &ifx_dev->mdm_reset_state), - IFX_RESET_TIMEOUT); - if (!ret) - dev_warn(&ifx_dev->spi_dev->dev, "Modem reset timeout: (state:%lx)", - ifx_dev->mdm_reset_state); - - ifx_dev->mdm_reset_state = 0; - return ret; -} - -/** - * ifx_spi_spi_probe - probe callback - * @spi: our possible matching SPI device - * - * Probe for a 6x60 modem on SPI bus. Perform any needed device and - * GPIO setup. - * - * FIXME: - * - Support for multiple devices - * - Split out MID specific GPIO handling eventually - */ - -static int ifx_spi_spi_probe(struct spi_device *spi) -{ - int ret; - int srdy; - struct ifx_modem_platform_data *pl_data; - struct ifx_spi_device *ifx_dev; - struct device *dev = &spi->dev; - - if (saved_ifx_dev) { - dev_dbg(dev, "ignoring subsequent detection"); - return -ENODEV; - } - - pl_data = dev_get_platdata(dev); - if (!pl_data) { - dev_err(dev, "missing platform data!"); - return -ENODEV; - } - - /* initialize structure to hold our device variables */ - ifx_dev = kzalloc(sizeof(struct ifx_spi_device), GFP_KERNEL); - if (!ifx_dev) { - dev_err(dev, "spi device allocation failed"); - return -ENOMEM; - } - saved_ifx_dev = ifx_dev; - ifx_dev->spi_dev = spi; - clear_bit(IFX_SPI_STATE_IO_IN_PROGRESS, &ifx_dev->flags); - spin_lock_init(&ifx_dev->write_lock); - spin_lock_init(&ifx_dev->power_lock); - ifx_dev->power_status = 0; - timer_setup(&ifx_dev->spi_timer, ifx_spi_timeout, 0); - ifx_dev->modem = pl_data->modem_type; - ifx_dev->use_dma = pl_data->use_dma; - ifx_dev->max_hz = pl_data->max_hz; - /* initialize spi mode, etc */ - spi->max_speed_hz = ifx_dev->max_hz; - spi->mode = IFX_SPI_MODE | (SPI_LOOP & spi->mode); - spi->bits_per_word = spi_bpw; - ret = spi_setup(spi); - if (ret) { - dev_err(dev, "SPI setup wasn't successful %d", ret); - kfree(ifx_dev); - return -ENODEV; - } - - /* init swap_buf function according to word width configuration */ - if (spi->bits_per_word == 32) - ifx_dev->swap_buf = swap_buf_32; - else if (spi->bits_per_word == 16) - ifx_dev->swap_buf = swap_buf_16; - else - ifx_dev->swap_buf = swap_buf_8; - - /* ensure SPI protocol flags are initialized to enable transfer */ - ifx_dev->spi_more = 0; - ifx_dev->spi_slave_cts = 0; - - /*initialize transfer and dma buffers */ - ifx_dev->tx_buffer = dma_alloc_coherent(ifx_dev->spi_dev->dev.parent, - IFX_SPI_TRANSFER_SIZE, - &ifx_dev->tx_bus, - GFP_KERNEL); - if (!ifx_dev->tx_buffer) { - dev_err(dev, "DMA-TX buffer allocation failed"); - ret = -ENOMEM; - goto error_ret; - } - ifx_dev->rx_buffer = dma_alloc_coherent(ifx_dev->spi_dev->dev.parent, - IFX_SPI_TRANSFER_SIZE, - &ifx_dev->rx_bus, - GFP_KERNEL); - if (!ifx_dev->rx_buffer) { - dev_err(dev, "DMA-RX buffer allocation failed"); - ret = -ENOMEM; - goto error_ret; - } - - /* initialize waitq for modem reset */ - init_waitqueue_head(&ifx_dev->mdm_reset_wait); - - spi_set_drvdata(spi, ifx_dev); - tasklet_setup(&ifx_dev->io_work_tasklet, ifx_spi_io); - - set_bit(IFX_SPI_STATE_PRESENT, &ifx_dev->flags); - - /* create our tty port */ - ret = ifx_spi_create_port(ifx_dev); - if (ret != 0) { - dev_err(dev, "create default tty port failed"); - goto error_ret; - } - - ifx_dev->gpio.reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW); - if (IS_ERR(ifx_dev->gpio.reset)) { - dev_err(dev, "could not obtain reset GPIO\n"); - ret = PTR_ERR(ifx_dev->gpio.reset); - goto error_ret; - } - gpiod_set_consumer_name(ifx_dev->gpio.reset, "ifxModem reset"); - ifx_dev->gpio.po = devm_gpiod_get(dev, "power", GPIOD_OUT_LOW); - if (IS_ERR(ifx_dev->gpio.po)) { - dev_err(dev, "could not obtain power GPIO\n"); - ret = PTR_ERR(ifx_dev->gpio.po); - goto error_ret; - } - gpiod_set_consumer_name(ifx_dev->gpio.po, "ifxModem power"); - ifx_dev->gpio.mrdy = devm_gpiod_get(dev, "mrdy", GPIOD_OUT_LOW); - if (IS_ERR(ifx_dev->gpio.mrdy)) { - dev_err(dev, "could not obtain mrdy GPIO\n"); - ret = PTR_ERR(ifx_dev->gpio.mrdy); - goto error_ret; - } - gpiod_set_consumer_name(ifx_dev->gpio.mrdy, "ifxModem mrdy"); - ifx_dev->gpio.srdy = devm_gpiod_get(dev, "srdy", GPIOD_IN); - if (IS_ERR(ifx_dev->gpio.srdy)) { - dev_err(dev, "could not obtain srdy GPIO\n"); - ret = PTR_ERR(ifx_dev->gpio.srdy); - goto error_ret; - } - gpiod_set_consumer_name(ifx_dev->gpio.srdy, "ifxModem srdy"); - ifx_dev->gpio.reset_out = devm_gpiod_get(dev, "rst_out", GPIOD_IN); - if (IS_ERR(ifx_dev->gpio.reset_out)) { - dev_err(dev, "could not obtain rst_out GPIO\n"); - ret = PTR_ERR(ifx_dev->gpio.reset_out); - goto error_ret; - } - gpiod_set_consumer_name(ifx_dev->gpio.reset_out, "ifxModem reset out"); - ifx_dev->gpio.pmu_reset = devm_gpiod_get(dev, "pmu_reset", GPIOD_ASIS); - if (IS_ERR(ifx_dev->gpio.pmu_reset)) { - dev_err(dev, "could not obtain pmu_reset GPIO\n"); - ret = PTR_ERR(ifx_dev->gpio.pmu_reset); - goto error_ret; - } - gpiod_set_consumer_name(ifx_dev->gpio.pmu_reset, "ifxModem PMU reset"); - - ret = request_irq(gpiod_to_irq(ifx_dev->gpio.reset_out), - ifx_spi_reset_interrupt, - IRQF_TRIGGER_RISING|IRQF_TRIGGER_FALLING, DRVNAME, - ifx_dev); - if (ret) { - dev_err(dev, "Unable to get irq %x\n", - gpiod_to_irq(ifx_dev->gpio.reset_out)); - goto error_ret; - } - - ret = ifx_spi_reset(ifx_dev); - - ret = request_irq(gpiod_to_irq(ifx_dev->gpio.srdy), - ifx_spi_srdy_interrupt, IRQF_TRIGGER_RISING, DRVNAME, - ifx_dev); - if (ret) { - dev_err(dev, "Unable to get irq %x", - gpiod_to_irq(ifx_dev->gpio.srdy)); - goto error_ret2; - } - - /* set pm runtime power state and register with power system */ - pm_runtime_set_active(dev); - pm_runtime_enable(dev); - - /* handle case that modem is already signaling SRDY */ - /* no outgoing tty open at this point, this just satisfies the - * modem's read and should reset communication properly - */ - srdy = gpiod_get_value(ifx_dev->gpio.srdy); - - if (srdy) { - mrdy_assert(ifx_dev); - ifx_spi_handle_srdy(ifx_dev); - } else - mrdy_set_low(ifx_dev); - return 0; - -error_ret2: - free_irq(gpiod_to_irq(ifx_dev->gpio.reset_out), ifx_dev); -error_ret: - ifx_spi_free_device(ifx_dev); - saved_ifx_dev = NULL; - return ret; -} - -/** - * ifx_spi_spi_remove - SPI device was removed - * @spi: SPI device - * - * FIXME: We should be shutting the device down here not in - * the module unload path. - */ - -static int ifx_spi_spi_remove(struct spi_device *spi) -{ - struct ifx_spi_device *ifx_dev = spi_get_drvdata(spi); - /* stop activity */ - tasklet_kill(&ifx_dev->io_work_tasklet); - - pm_runtime_disable(&spi->dev); - - /* free irq */ - free_irq(gpiod_to_irq(ifx_dev->gpio.reset_out), ifx_dev); - free_irq(gpiod_to_irq(ifx_dev->gpio.srdy), ifx_dev); - - /* free allocations */ - ifx_spi_free_device(ifx_dev); - - saved_ifx_dev = NULL; - return 0; -} - -/** - * ifx_spi_spi_shutdown - called on SPI shutdown - * @spi: SPI device - * - * No action needs to be taken here - */ - -static void ifx_spi_spi_shutdown(struct spi_device *spi) -{ - struct ifx_spi_device *ifx_dev = spi_get_drvdata(spi); - - ifx_modem_power_off(ifx_dev); -} - -/* - * various suspends and resumes have nothing to do - * no hardware to save state for - */ - -/** - * ifx_spi_pm_suspend - suspend modem on system suspend - * @dev: device being suspended - * - * Suspend the modem. No action needed on Intel MID platforms, may - * need extending for other systems. - */ -static int ifx_spi_pm_suspend(struct device *dev) -{ - return 0; -} - -/** - * ifx_spi_pm_resume - resume modem on system resume - * @dev: device being suspended - * - * Allow the modem to resume. No action needed. - * - * FIXME: do we need to reset anything here ? - */ -static int ifx_spi_pm_resume(struct device *dev) -{ - return 0; -} - -/** - * ifx_spi_pm_runtime_resume - suspend modem - * @dev: device being suspended - * - * Allow the modem to resume. No action needed. - */ -static int ifx_spi_pm_runtime_resume(struct device *dev) -{ - return 0; -} - -/** - * ifx_spi_pm_runtime_suspend - suspend modem - * @dev: device being suspended - * - * Allow the modem to suspend and thus suspend to continue up the - * device tree. - */ -static int ifx_spi_pm_runtime_suspend(struct device *dev) -{ - return 0; -} - -/** - * ifx_spi_pm_runtime_idle - check if modem idle - * @dev: our device - * - * Check conditions and queue runtime suspend if idle. - */ -static int ifx_spi_pm_runtime_idle(struct device *dev) -{ - struct spi_device *spi = to_spi_device(dev); - struct ifx_spi_device *ifx_dev = spi_get_drvdata(spi); - - if (!ifx_dev->power_status) - pm_runtime_suspend(dev); - - return 0; -} - -static const struct dev_pm_ops ifx_spi_pm = { - .resume = ifx_spi_pm_resume, - .suspend = ifx_spi_pm_suspend, - .runtime_resume = ifx_spi_pm_runtime_resume, - .runtime_suspend = ifx_spi_pm_runtime_suspend, - .runtime_idle = ifx_spi_pm_runtime_idle -}; - -static const struct spi_device_id ifx_id_table[] = { - {"ifx6160", 0}, - {"ifx6260", 0}, - { } -}; -MODULE_DEVICE_TABLE(spi, ifx_id_table); - -/* spi operations */ -static struct spi_driver ifx_spi_driver = { - .driver = { - .name = DRVNAME, - .pm = &ifx_spi_pm, - }, - .probe = ifx_spi_spi_probe, - .shutdown = ifx_spi_spi_shutdown, - .remove = ifx_spi_spi_remove, - .id_table = ifx_id_table -}; - -/** - * ifx_spi_exit - module exit - * - * Unload the module. - */ - -static void __exit ifx_spi_exit(void) -{ - /* unregister */ - spi_unregister_driver(&ifx_spi_driver); - tty_unregister_driver(tty_drv); - put_tty_driver(tty_drv); - unregister_reboot_notifier(&ifx_modem_reboot_notifier_block); -} - -/** - * ifx_spi_init - module entry point - * - * Initialise the SPI and tty interfaces for the IFX SPI driver - * We need to initialize upper-edge spi driver after the tty - * driver because otherwise the spi probe will race - */ - -static int __init ifx_spi_init(void) -{ - int result; - - tty_drv = alloc_tty_driver(1); - if (!tty_drv) { - pr_err("%s: alloc_tty_driver failed", DRVNAME); - return -ENOMEM; - } - - tty_drv->driver_name = DRVNAME; - tty_drv->name = TTYNAME; - tty_drv->minor_start = IFX_SPI_TTY_ID; - tty_drv->type = TTY_DRIVER_TYPE_SERIAL; - tty_drv->subtype = SERIAL_TYPE_NORMAL; - tty_drv->flags = TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV; - tty_drv->init_termios = tty_std_termios; - - tty_set_operations(tty_drv, &ifx_spi_serial_ops); - - result = tty_register_driver(tty_drv); - if (result) { - pr_err("%s: tty_register_driver failed(%d)", - DRVNAME, result); - goto err_free_tty; - } - - result = spi_register_driver(&ifx_spi_driver); - if (result) { - pr_err("%s: spi_register_driver failed(%d)", - DRVNAME, result); - goto err_unreg_tty; - } - - result = register_reboot_notifier(&ifx_modem_reboot_notifier_block); - if (result) { - pr_err("%s: register ifx modem reboot notifier failed(%d)", - DRVNAME, result); - goto err_unreg_spi; - } - - return 0; -err_unreg_spi: - spi_unregister_driver(&ifx_spi_driver); -err_unreg_tty: - tty_unregister_driver(tty_drv); -err_free_tty: - put_tty_driver(tty_drv); - - return result; -} - -module_init(ifx_spi_init); -module_exit(ifx_spi_exit); - -MODULE_AUTHOR("Intel"); -MODULE_DESCRIPTION("IFX6x60 spi driver"); -MODULE_LICENSE("GPL"); -MODULE_INFO(Version, "0.1-IFX6x60"); diff --git a/drivers/tty/serial/ifx6x60.h b/drivers/tty/serial/ifx6x60.h deleted file mode 100644 index ecb841d928a7..000000000000 --- a/drivers/tty/serial/ifx6x60.h +++ /dev/null @@ -1,118 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/**************************************************************************** - * - * Driver for the IFX spi modem. - * - * Copyright (C) 2009, 2010 Intel Corp - * Jim Stanley - * - *****************************************************************************/ -#ifndef _IFX6X60_H -#define _IFX6X60_H - -struct gpio_desc; - -#define DRVNAME "ifx6x60" -#define TTYNAME "ttyIFX" - -#define IFX_SPI_MAX_MINORS 1 -#define IFX_SPI_TRANSFER_SIZE 2048 -#define IFX_SPI_FIFO_SIZE 4096 - -#define IFX_SPI_HEADER_OVERHEAD 4 -#define IFX_RESET_TIMEOUT msecs_to_jiffies(50) - -/* device flags bitfield definitions */ -#define IFX_SPI_STATE_PRESENT 0 -#define IFX_SPI_STATE_IO_IN_PROGRESS 1 -#define IFX_SPI_STATE_IO_READY 2 -#define IFX_SPI_STATE_TIMER_PENDING 3 -#define IFX_SPI_STATE_IO_AVAILABLE 4 - -/* flow control bitfields */ -#define IFX_SPI_DCD 0 -#define IFX_SPI_CTS 1 -#define IFX_SPI_DSR 2 -#define IFX_SPI_RI 3 -#define IFX_SPI_DTR 4 -#define IFX_SPI_RTS 5 -#define IFX_SPI_TX_FC 6 -#define IFX_SPI_RX_FC 7 -#define IFX_SPI_UPDATE 8 - -#define IFX_SPI_PAYLOAD_SIZE (IFX_SPI_TRANSFER_SIZE - \ - IFX_SPI_HEADER_OVERHEAD) - -#define IFX_SPI_IRQ_TYPE DETECT_EDGE_RISING -#define IFX_SPI_GPIO_TARGET 0 -#define IFX_SPI_GPIO0 0x105 - -#define IFX_SPI_STATUS_TIMEOUT (2000*HZ) - -/* values for bits in power status byte */ -#define IFX_SPI_POWER_DATA_PENDING 1 -#define IFX_SPI_POWER_SRDY 2 - -struct ifx_spi_device { - /* Our SPI device */ - struct spi_device *spi_dev; - - /* Port specific data */ - struct kfifo tx_fifo; - spinlock_t fifo_lock; - unsigned long signal_state; - - /* TTY Layer logic */ - struct tty_port tty_port; - struct device *tty_dev; - int minor; - - /* Low level I/O work */ - struct tasklet_struct io_work_tasklet; - unsigned long flags; - dma_addr_t rx_dma; - dma_addr_t tx_dma; - - int modem; /* Modem type */ - int use_dma; /* provide dma-able addrs in SPI msg */ - long max_hz; /* max SPI frequency */ - - spinlock_t write_lock; - int write_pending; - spinlock_t power_lock; - unsigned char power_status; - - unsigned char *rx_buffer; - unsigned char *tx_buffer; - dma_addr_t rx_bus; - dma_addr_t tx_bus; - unsigned char spi_more; - unsigned char spi_slave_cts; - - struct timer_list spi_timer; - - struct spi_message spi_msg; - struct spi_transfer spi_xfer; - - struct { - /* gpio lines */ - struct gpio_desc *srdy; /* slave-ready gpio */ - struct gpio_desc *mrdy; /* master-ready gpio */ - struct gpio_desc *reset; /* modem-reset gpio */ - struct gpio_desc *po; /* modem-on gpio */ - struct gpio_desc *reset_out; /* modem-in-reset gpio */ - struct gpio_desc *pmu_reset; /* PMU reset gpio */ - /* state/stats */ - int unack_srdy_int_nb; - } gpio; - - /* modem reset */ - unsigned long mdm_reset_state; -#define MR_START 0 -#define MR_INPROGRESS 1 -#define MR_COMPLETE 2 - wait_queue_head_t mdm_reset_wait; - void (*swap_buf)(unsigned char *buf, int len, void *end); -}; - -#endif /* _IFX6X60_H */ diff --git a/include/linux/spi/ifx_modem.h b/include/linux/spi/ifx_modem.h deleted file mode 100644 index 6d19b09139d0..000000000000 --- a/include/linux/spi/ifx_modem.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef LINUX_IFX_MODEM_H -#define LINUX_IFX_MODEM_H - -struct ifx_modem_platform_data { - unsigned short tx_pwr; /* modem power threshold */ - unsigned char modem_type; /* Modem type */ - unsigned long max_hz; /* max SPI frequency */ - unsigned short use_dma:1; /* spi protocol driver supplies - dma-able addrs */ -}; -#define IFX_MODEM_6160 1 -#define IFX_MODEM_6260 2 - -#endif -- cgit v1.2.3 From 2c2b9fd6b496b3616e9b9537ea0258b3040914f3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 9 Jan 2021 12:13:32 +0100 Subject: block: unexport truncate_bdev_range truncate_bdev_range is only used in always built-in block layer code, so remove the export and the !CONFIG_BLOCK stub. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- fs/block_dev.c | 1 - include/linux/blkdev.h | 9 ++------- 2 files changed, 2 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/fs/block_dev.c b/fs/block_dev.c index 289c3dd923a4..c1fe29dac485 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -126,7 +126,6 @@ int truncate_bdev_range(struct block_device *bdev, fmode_t mode, bd_abort_claiming(bdev, truncate_bdev_range); return 0; } -EXPORT_SYMBOL(truncate_bdev_range); static void set_init_blocksize(struct block_device *bdev) { diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 20f3706b6b2e..2491e17b61c4 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1999,21 +1999,16 @@ void bdev_add(struct block_device *bdev, dev_t dev); struct block_device *I_BDEV(struct inode *inode); struct block_device *bdgrab(struct block_device *bdev); void bdput(struct block_device *); +int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart, + loff_t lend); #ifdef CONFIG_BLOCK void invalidate_bdev(struct block_device *bdev); -int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart, - loff_t lend); int sync_blockdev(struct block_device *bdev); #else static inline void invalidate_bdev(struct block_device *bdev) { } -static inline int truncate_bdev_range(struct block_device *bdev, fmode_t mode, - loff_t lstart, loff_t lend) -{ - return 0; -} static inline int sync_blockdev(struct block_device *bdev) { return 0; -- cgit v1.2.3 From 6b2e04bc240fe9be9e690059f710e9f95346d34d Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Mon, 25 Jan 2021 13:44:30 -0800 Subject: net: allow user to set metric on default route learned via Router Advertisement For IPv4, default route is learned via DHCPv4 and user is allowed to change metric using config etc/network/interfaces. But for IPv6, default route can be learned via RA, for which, currently a fixed metric value 1024 is used. Ideally, user should be able to configure metric on default route for IPv6 similar to IPv4. This patch adds sysctl for the same. Logs: For IPv4: Config in etc/network/interfaces: auto eth0 iface eth0 inet dhcp metric 4261413864 IPv4 Kernel Route Table: $ ip route list default via 172.21.47.1 dev eth0 metric 4261413864 FRR Table, if a static route is configured: [In real scenario, it is useful to prefer BGP learned default route over DHCPv4 default route.] Codes: K - kernel route, C - connected, S - static, R - RIP, O - OSPF, I - IS-IS, B - BGP, P - PIM, E - EIGRP, N - NHRP, T - Table, v - VNC, V - VNC-Direct, A - Babel, D - SHARP, > - selected route, * - FIB route S>* 0.0.0.0/0 [20/0] is directly connected, eth0, 00:00:03 K 0.0.0.0/0 [254/1000] via 172.21.47.1, eth0, 6d08h51m i.e. User can prefer Default Router learned via Routing Protocol in IPv4. Similar behavior is not possible for IPv6, without this fix. After fix [for IPv6]: sudo sysctl -w net.ipv6.conf.eth0.net.ipv6.conf.eth0.ra_defrtr_metric=1996489705 IP monitor: [When IPv6 RA is received] default via fe80::xx16:xxxx:feb3:ce8e dev eth0 proto ra metric 1996489705 pref high Kernel IPv6 routing table $ ip -6 route list default via fe80::be16:65ff:feb3:ce8e dev eth0 proto ra metric 1996489705 expires 21sec hoplimit 64 pref high FRR Table, if a static route is configured: [In real scenario, it is useful to prefer BGP learned default route over IPv6 RA default route.] Codes: K - kernel route, C - connected, S - static, R - RIPng, O - OSPFv3, I - IS-IS, B - BGP, N - NHRP, T - Table, v - VNC, V - VNC-Direct, A - Babel, D - SHARP, > - selected route, * - FIB route S>* ::/0 [20/0] is directly connected, eth0, 00:00:06 K ::/0 [119/1001] via fe80::xx16:xxxx:feb3:ce8e, eth0, 6d07h43m If the metric is changed later, the effect will be seen only when next IPv6 RA is received, because the default route must be fully controlled by RA msg. Below metric is changed from 1996489705 to 1996489704. $ sudo sysctl -w net.ipv6.conf.eth0.ra_defrtr_metric=1996489704 net.ipv6.conf.eth0.ra_defrtr_metric = 1996489704 IP monitor: [On next IPv6 RA msg, Kernel deletes prev route and installs new route with updated metric] Deleted default via fe80::xx16:xxxx:feb3:ce8e dev eth0 proto ra metric 1996489705 expires 3sec hoplimit 64 pref high default via fe80::xx16:xxxx:feb3:ce8e dev eth0 proto ra metric 1996489704 pref high Signed-off-by: Praveen Chaudhary Signed-off-by: Zhenggen Xu Reviewed-by: David Ahern Link: https://lore.kernel.org/r/20210125214430.24079-1-pchaudhary@linkedin.com Signed-off-by: Jakub Kicinski --- Documentation/networking/ip-sysctl.rst | 10 ++++++++++ include/linux/ipv6.h | 1 + include/net/ip6_route.h | 3 ++- include/uapi/linux/ipv6.h | 1 + include/uapi/linux/sysctl.h | 1 + net/ipv6/addrconf.c | 11 +++++++++++ net/ipv6/ndisc.c | 12 ++++++++---- net/ipv6/route.c | 5 +++-- 8 files changed, 37 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index dd2b12a32b73..0e51ddd9a2f1 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -1871,6 +1871,16 @@ accept_ra_defrtr - BOOLEAN - enabled if accept_ra is enabled. - disabled if accept_ra is disabled. +ra_defrtr_metric - UNSIGNED INTEGER + Route metric for default route learned in Router Advertisement. This value + will be assigned as metric for the default route learned via IPv6 Router + Advertisement. Takes affect only if accept_ra_defrtr is enabled. + + Possible values: + 1 to 0xFFFFFFFF + + Default: IP6_RT_PRIO_USER i.e. 1024. + accept_ra_from_local - BOOLEAN Accept RA with source-address that is found on local machine if the RA is otherwise proper and able to be accepted. diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index dda61d150a13..9d1f29f0c512 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -31,6 +31,7 @@ struct ipv6_devconf { __s32 max_desync_factor; __s32 max_addresses; __s32 accept_ra_defrtr; + __u32 ra_defrtr_metric; __s32 accept_ra_min_hop_limit; __s32 accept_ra_pinfo; __s32 ignore_routes_with_linkdown; diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index 2a5277758379..f51a118bfce8 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -174,7 +174,8 @@ struct fib6_info *rt6_get_dflt_router(struct net *net, struct net_device *dev); struct fib6_info *rt6_add_dflt_router(struct net *net, const struct in6_addr *gwaddr, - struct net_device *dev, unsigned int pref); + struct net_device *dev, unsigned int pref, + u32 defrtr_usr_metric); void rt6_purge_dflt_routers(struct net *net); diff --git a/include/uapi/linux/ipv6.h b/include/uapi/linux/ipv6.h index 13e8751bf24a..70603775fe91 100644 --- a/include/uapi/linux/ipv6.h +++ b/include/uapi/linux/ipv6.h @@ -189,6 +189,7 @@ enum { DEVCONF_ACCEPT_RA_RT_INFO_MIN_PLEN, DEVCONF_NDISC_TCLASS, DEVCONF_RPL_SEG_ENABLED, + DEVCONF_RA_DEFRTR_METRIC, DEVCONF_MAX }; diff --git a/include/uapi/linux/sysctl.h b/include/uapi/linux/sysctl.h index 458179df9b27..1e05d3caa712 100644 --- a/include/uapi/linux/sysctl.h +++ b/include/uapi/linux/sysctl.h @@ -571,6 +571,7 @@ enum { NET_IPV6_ACCEPT_SOURCE_ROUTE=25, NET_IPV6_ACCEPT_RA_FROM_LOCAL=26, NET_IPV6_ACCEPT_RA_RT_INFO_MIN_PLEN=27, + NET_IPV6_RA_DEFRTR_METRIC=28, __NET_IPV6_MAX }; diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 9edc5bb2d531..f2337fb756ac 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -205,6 +205,7 @@ static struct ipv6_devconf ipv6_devconf __read_mostly = { .max_desync_factor = MAX_DESYNC_FACTOR, .max_addresses = IPV6_MAX_ADDRESSES, .accept_ra_defrtr = 1, + .ra_defrtr_metric = IP6_RT_PRIO_USER, .accept_ra_from_local = 0, .accept_ra_min_hop_limit= 1, .accept_ra_pinfo = 1, @@ -260,6 +261,7 @@ static struct ipv6_devconf ipv6_devconf_dflt __read_mostly = { .max_desync_factor = MAX_DESYNC_FACTOR, .max_addresses = IPV6_MAX_ADDRESSES, .accept_ra_defrtr = 1, + .ra_defrtr_metric = IP6_RT_PRIO_USER, .accept_ra_from_local = 0, .accept_ra_min_hop_limit= 1, .accept_ra_pinfo = 1, @@ -5476,6 +5478,7 @@ static inline void ipv6_store_devconf(struct ipv6_devconf *cnf, array[DEVCONF_MAX_DESYNC_FACTOR] = cnf->max_desync_factor; array[DEVCONF_MAX_ADDRESSES] = cnf->max_addresses; array[DEVCONF_ACCEPT_RA_DEFRTR] = cnf->accept_ra_defrtr; + array[DEVCONF_RA_DEFRTR_METRIC] = cnf->ra_defrtr_metric; array[DEVCONF_ACCEPT_RA_MIN_HOP_LIMIT] = cnf->accept_ra_min_hop_limit; array[DEVCONF_ACCEPT_RA_PINFO] = cnf->accept_ra_pinfo; #ifdef CONFIG_IPV6_ROUTER_PREF @@ -6668,6 +6671,14 @@ static const struct ctl_table addrconf_sysctl[] = { .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "ra_defrtr_metric", + .data = &ipv6_devconf.ra_defrtr_metric, + .maxlen = sizeof(u32), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = (void *)SYSCTL_ONE, + }, { .procname = "accept_ra_min_hop_limit", .data = &ipv6_devconf.accept_ra_min_hop_limit, diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c index 76717478f173..c467c6419893 100644 --- a/net/ipv6/ndisc.c +++ b/net/ipv6/ndisc.c @@ -1173,6 +1173,7 @@ static void ndisc_router_discovery(struct sk_buff *skb) struct neighbour *neigh = NULL; struct inet6_dev *in6_dev; struct fib6_info *rt = NULL; + u32 defrtr_usr_metric; struct net *net; int lifetime; struct ndisc_options ndopts; @@ -1303,18 +1304,21 @@ static void ndisc_router_discovery(struct sk_buff *skb) return; } } - if (rt && lifetime == 0) { + /* Set default route metric as specified by user */ + defrtr_usr_metric = in6_dev->cnf.ra_defrtr_metric; + /* delete the route if lifetime is 0 or if metric needs change */ + if (rt && (lifetime == 0 || rt->fib6_metric != defrtr_usr_metric)) { ip6_del_rt(net, rt, false); rt = NULL; } - ND_PRINTK(3, info, "RA: rt: %p lifetime: %d, for dev: %s\n", - rt, lifetime, skb->dev->name); + ND_PRINTK(3, info, "RA: rt: %p lifetime: %d, metric: %d, for dev: %s\n", + rt, lifetime, defrtr_usr_metric, skb->dev->name); if (!rt && lifetime) { ND_PRINTK(3, info, "RA: adding default router\n"); rt = rt6_add_dflt_router(net, &ipv6_hdr(skb)->saddr, - skb->dev, pref); + skb->dev, pref, defrtr_usr_metric); if (!rt) { ND_PRINTK(0, err, "RA: %s failed to add default route\n", diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 188e114b29b4..41d8f801b75f 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -4252,11 +4252,12 @@ struct fib6_info *rt6_get_dflt_router(struct net *net, struct fib6_info *rt6_add_dflt_router(struct net *net, const struct in6_addr *gwaddr, struct net_device *dev, - unsigned int pref) + unsigned int pref, + u32 defrtr_usr_metric) { struct fib6_config cfg = { .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT, - .fc_metric = IP6_RT_PRIO_USER, + .fc_metric = defrtr_usr_metric, .fc_ifindex = dev->ifindex, .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT | RTF_UP | RTF_EXPIRES | RTF_PREF(pref), -- cgit v1.2.3 From 6fe27d68b45666381691b6a841a2adbda8c8d153 Mon Sep 17 00:00:00 2001 From: Vincent Mailhol Date: Wed, 20 Jan 2021 02:03:55 +0900 Subject: can: dev: export can_get_state_str() function The can_get_state_str() function is also relevant to the drivers. Export the symbol and make it visible in the can/dev.h header. Link: https://lore.kernel.org/r/20210119170355.12040-1-mailhol.vincent@wanadoo.fr Signed-off-by: Vincent Mailhol Signed-off-by: Marc Kleine-Budde --- drivers/net/can/dev/dev.c | 3 ++- include/linux/can/dev.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/net/can/dev/dev.c b/drivers/net/can/dev/dev.c index 01e4a194f187..d9281ae853f8 100644 --- a/drivers/net/can/dev/dev.c +++ b/drivers/net/can/dev/dev.c @@ -74,7 +74,7 @@ static int can_rx_state_to_frame(struct net_device *dev, enum can_state state) } } -static const char *can_get_state_str(const enum can_state state) +const char *can_get_state_str(const enum can_state state) { switch (state) { case CAN_STATE_ERROR_ACTIVE: @@ -95,6 +95,7 @@ static const char *can_get_state_str(const enum can_state state) return ""; } +EXPORT_SYMBOL_GPL(can_get_state_str); void can_change_state(struct net_device *dev, struct can_frame *cf, enum can_state tx_state, enum can_state rx_state) diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h index 7faf6a37d5b2..ac4d83a1ab81 100644 --- a/include/linux/can/dev.h +++ b/include/linux/can/dev.h @@ -123,6 +123,7 @@ void unregister_candev(struct net_device *dev); int can_restart_now(struct net_device *dev); void can_bus_off(struct net_device *dev); +const char *can_get_state_str(const enum can_state state); void can_change_state(struct net_device *dev, struct can_frame *cf, enum can_state tx_state, enum can_state rx_state); -- cgit v1.2.3 From 87baa23e0236bc1c41ce744f524bf12dbe7fde66 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Mon, 11 Jan 2021 19:07:40 +0100 Subject: bus: mhi: core: Add helper API to return number of free TREs Introduce mhi_get_free_desc_count() API to return number of TREs available to queue buffer. MHI clients can use this API to know before hand if ring is full without calling queue API. Signed-off-by: Hemant Kumar Reviewed-by: Jeffrey Hugo Reviewed-by: Manivannan Sadhasivam Link: https://lore.kernel.org/r/1610388462-16322-1-git-send-email-loic.poulain@linaro.org Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/core/main.c | 12 ++++++++++++ include/linux/mhi.h | 9 +++++++++ 2 files changed, 21 insertions(+) (limited to 'include/linux') diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c index d34d7e90e38d..1202433ecf98 100644 --- a/drivers/bus/mhi/core/main.c +++ b/drivers/bus/mhi/core/main.c @@ -260,6 +260,18 @@ int mhi_destroy_device(struct device *dev, void *data) return 0; } +int mhi_get_free_desc_count(struct mhi_device *mhi_dev, + enum dma_data_direction dir) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? + mhi_dev->ul_chan : mhi_dev->dl_chan; + struct mhi_ring *tre_ring = &mhi_chan->tre_ring; + + return get_nr_avail_ring_elements(mhi_cntrl, tre_ring); +} +EXPORT_SYMBOL_GPL(mhi_get_free_desc_count); + void mhi_notify(struct mhi_device *mhi_dev, enum mhi_callback cb_reason) { struct mhi_driver *mhi_drv; diff --git a/include/linux/mhi.h b/include/linux/mhi.h index 562862ff819c..ece53a252217 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -598,6 +598,15 @@ void mhi_set_mhi_state(struct mhi_controller *mhi_cntrl, */ void mhi_notify(struct mhi_device *mhi_dev, enum mhi_callback cb_reason); +/** + * mhi_get_free_desc_count - Get transfer ring length + * Get # of TD available to queue buffers + * @mhi_dev: Device associated with the channels + * @dir: Direction of the channel + */ +int mhi_get_free_desc_count(struct mhi_device *mhi_dev, + enum dma_data_direction dir); + /** * mhi_prepare_for_power_up - Do pre-initialization before power up. * This is optional, call this before power up if -- cgit v1.2.3 From 620a6dc40754dc218f5b6389b5d335e9a107fd29 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Fri, 22 Jan 2021 12:39:43 +0000 Subject: sched/topology: Make sched_init_numa() use a set for the deduplicating sort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deduplicating sort in sched_init_numa() assumes that the first line in the distance table contains all unique values in the entire table. I've been trying to pen what this exactly means for the topology, but it's not straightforward. For instance, topology.c uses this example: node 0 1 2 3 0: 10 20 20 30 1: 20 10 20 20 2: 20 20 10 20 3: 30 20 20 10 0 ----- 1 | / | | / | | / | 2 ----- 3 Which works out just fine. However, if we swap nodes 0 and 1: 1 ----- 0 | / | | / | | / | 2 ----- 3 we get this distance table: node 0 1 2 3 0: 10 20 20 20 1: 20 10 20 30 2: 20 20 10 20 3: 20 30 20 10 Which breaks the deduplicating sort (non-representative first line). In this case this would just be a renumbering exercise, but it so happens that we can have a deduplicating sort that goes through the whole table in O(n²) at the extra cost of a temporary memory allocation (i.e. any form of set). The ACPI spec (SLIT) mentions distances are encoded on 8 bits. Following this, implement the set as a 256-bits bitmap. Should this not be satisfactory (i.e. we want to support 32-bit values), then we'll have to go for some other sparse set implementation. This has the added benefit of letting us allocate just the right amount of memory for sched_domains_numa_distance[], rather than an arbitrary (nr_node_ids + 1). Note: DT binding equivalent (distance-map) decodes distances as 32-bit values. Signed-off-by: Valentin Schneider Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20210122123943.1217-2-valentin.schneider@arm.com --- include/linux/topology.h | 1 + kernel/sched/topology.c | 99 +++++++++++++++++++++++------------------------- 2 files changed, 49 insertions(+), 51 deletions(-) (limited to 'include/linux') diff --git a/include/linux/topology.h b/include/linux/topology.h index ad03df1cc266..7634cd737061 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -48,6 +48,7 @@ int arch_update_cpu_topology(void); /* Conform to ACPI 2.0 SLIT distance definitions */ #define LOCAL_DISTANCE 10 #define REMOTE_DISTANCE 20 +#define DISTANCE_BITS 8 #ifndef node_distance #define node_distance(from,to) ((from) == (to) ? LOCAL_DISTANCE : REMOTE_DISTANCE) #endif diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 5d3675c7a76b..bf5c9bd10bfe 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1596,66 +1596,58 @@ static void init_numa_topology_type(void) } } + +#define NR_DISTANCE_VALUES (1 << DISTANCE_BITS) + void sched_init_numa(void) { - int next_distance, curr_distance = node_distance(0, 0); struct sched_domain_topology_level *tl; - int level = 0; - int i, j, k; - - sched_domains_numa_distance = kzalloc(sizeof(int) * (nr_node_ids + 1), GFP_KERNEL); - if (!sched_domains_numa_distance) - return; - - /* Includes NUMA identity node at level 0. */ - sched_domains_numa_distance[level++] = curr_distance; - sched_domains_numa_levels = level; + unsigned long *distance_map; + int nr_levels = 0; + int i, j; /* * O(nr_nodes^2) deduplicating selection sort -- in order to find the * unique distances in the node_distance() table. - * - * Assumes node_distance(0,j) includes all distances in - * node_distance(i,j) in order to avoid cubic time. */ - next_distance = curr_distance; + distance_map = bitmap_alloc(NR_DISTANCE_VALUES, GFP_KERNEL); + if (!distance_map) + return; + + bitmap_zero(distance_map, NR_DISTANCE_VALUES); for (i = 0; i < nr_node_ids; i++) { for (j = 0; j < nr_node_ids; j++) { - for (k = 0; k < nr_node_ids; k++) { - int distance = node_distance(i, k); - - if (distance > curr_distance && - (distance < next_distance || - next_distance == curr_distance)) - next_distance = distance; - - /* - * While not a strong assumption it would be nice to know - * about cases where if node A is connected to B, B is not - * equally connected to A. - */ - if (sched_debug() && node_distance(k, i) != distance) - sched_numa_warn("Node-distance not symmetric"); + int distance = node_distance(i, j); - if (sched_debug() && i && !find_numa_distance(distance)) - sched_numa_warn("Node-0 not representative"); + if (distance < LOCAL_DISTANCE || distance >= NR_DISTANCE_VALUES) { + sched_numa_warn("Invalid distance value range"); + return; } - if (next_distance != curr_distance) { - sched_domains_numa_distance[level++] = next_distance; - sched_domains_numa_levels = level; - curr_distance = next_distance; - } else break; + + bitmap_set(distance_map, distance, 1); } + } + /* + * We can now figure out how many unique distance values there are and + * allocate memory accordingly. + */ + nr_levels = bitmap_weight(distance_map, NR_DISTANCE_VALUES); - /* - * In case of sched_debug() we verify the above assumption. - */ - if (!sched_debug()) - break; + sched_domains_numa_distance = kcalloc(nr_levels, sizeof(int), GFP_KERNEL); + if (!sched_domains_numa_distance) { + bitmap_free(distance_map); + return; + } + + for (i = 0, j = 0; i < nr_levels; i++, j++) { + j = find_next_bit(distance_map, NR_DISTANCE_VALUES, j); + sched_domains_numa_distance[i] = j; } + bitmap_free(distance_map); + /* - * 'level' contains the number of unique distances + * 'nr_levels' contains the number of unique distances * * The sched_domains_numa_distance[] array includes the actual distance * numbers. @@ -1664,15 +1656,15 @@ void sched_init_numa(void) /* * Here, we should temporarily reset sched_domains_numa_levels to 0. * If it fails to allocate memory for array sched_domains_numa_masks[][], - * the array will contain less then 'level' members. This could be + * the array will contain less then 'nr_levels' members. This could be * dangerous when we use it to iterate array sched_domains_numa_masks[][] * in other functions. * - * We reset it to 'level' at the end of this function. + * We reset it to 'nr_levels' at the end of this function. */ sched_domains_numa_levels = 0; - sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL); + sched_domains_numa_masks = kzalloc(sizeof(void *) * nr_levels, GFP_KERNEL); if (!sched_domains_numa_masks) return; @@ -1680,7 +1672,7 @@ void sched_init_numa(void) * Now for each level, construct a mask per node which contains all * CPUs of nodes that are that many hops away from us. */ - for (i = 0; i < level; i++) { + for (i = 0; i < nr_levels; i++) { sched_domains_numa_masks[i] = kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL); if (!sched_domains_numa_masks[i]) @@ -1688,12 +1680,17 @@ void sched_init_numa(void) for (j = 0; j < nr_node_ids; j++) { struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL); + int k; + if (!mask) return; sched_domains_numa_masks[i][j] = mask; for_each_node(k) { + if (sched_debug() && (node_distance(j, k) != node_distance(k, j))) + sched_numa_warn("Node-distance not symmetric"); + if (node_distance(j, k) > sched_domains_numa_distance[i]) continue; @@ -1705,7 +1702,7 @@ void sched_init_numa(void) /* Compute default topology size */ for (i = 0; sched_domain_topology[i].mask; i++); - tl = kzalloc((i + level + 1) * + tl = kzalloc((i + nr_levels) * sizeof(struct sched_domain_topology_level), GFP_KERNEL); if (!tl) return; @@ -1728,7 +1725,7 @@ void sched_init_numa(void) /* * .. and append 'j' levels of NUMA goodness. */ - for (j = 1; j < level; i++, j++) { + for (j = 1; j < nr_levels; i++, j++) { tl[i] = (struct sched_domain_topology_level){ .mask = sd_numa_mask, .sd_flags = cpu_numa_flags, @@ -1740,8 +1737,8 @@ void sched_init_numa(void) sched_domain_topology = tl; - sched_domains_numa_levels = level; - sched_max_numa_distance = sched_domains_numa_distance[level - 1]; + sched_domains_numa_levels = nr_levels; + sched_max_numa_distance = sched_domains_numa_distance[nr_levels - 1]; init_numa_topology_type(); } -- cgit v1.2.3 From 1875dc5b8ff4690547c446ef222083e28e2d9463 Mon Sep 17 00:00:00 2001 From: Peter Oskolkov Date: Tue, 26 Jan 2021 11:34:49 -0800 Subject: sched: Correctly sort struct predeclarations The comment says: /* task_struct member predeclarations (sorted alphabetically): */ So move io_uring_task where it belongs (alphabetically). Signed-off-by: Peter Oskolkov Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20210126193449.487547-1-posk@google.com --- include/linux/sched.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index 31169e70476a..e1152227bb79 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -47,6 +47,7 @@ struct cfs_rq; struct fs_struct; struct futex_pi_state; struct io_context; +struct io_uring_task; struct mempolicy; struct nameidata; struct nsproxy; @@ -65,7 +66,6 @@ struct sighand_struct; struct signal_struct; struct task_delay_info; struct task_group; -struct io_uring_task; /* * Task state bitmask. NOTE! These bits are also -- cgit v1.2.3 From 3175199ab0ac8c874ec25c6bf169f74888917435 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 26 Jan 2021 15:52:34 +0100 Subject: block: split bio_kmalloc from bio_alloc_bioset bio_kmalloc shares almost no logic with the bio_set based fast path in bio_alloc_bioset. Split it into an entirely separate implementation. Signed-off-by: Christoph Hellwig Reviewed-by: Johannes Thumshirn Reviewed-by: Chaitanya Kulkarni Acked-by: Damien Le Moal Signed-off-by: Jens Axboe --- block/bio.c | 167 ++++++++++++++++++++++++++-------------------------- include/linux/bio.h | 6 +- 2 files changed, 86 insertions(+), 87 deletions(-) (limited to 'include/linux') diff --git a/block/bio.c b/block/bio.c index dfd7740a3230..d4375619348c 100644 --- a/block/bio.c +++ b/block/bio.c @@ -396,123 +396,101 @@ static void punt_bios_to_rescuer(struct bio_set *bs) * @nr_iovecs: number of iovecs to pre-allocate * @bs: the bio_set to allocate from. * - * Description: - * If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is - * backed by the @bs's mempool. + * Allocate a bio from the mempools in @bs. * - * When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will - * always be able to allocate a bio. This is due to the mempool guarantees. - * To make this work, callers must never allocate more than 1 bio at a time - * from this pool. Callers that need to allocate more than 1 bio must always - * submit the previously allocated bio for IO before attempting to allocate - * a new one. Failure to do so can cause deadlocks under memory pressure. + * If %__GFP_DIRECT_RECLAIM is set then bio_alloc will always be able to + * allocate a bio. This is due to the mempool guarantees. To make this work, + * callers must never allocate more than 1 bio at a time from the general pool. + * Callers that need to allocate more than 1 bio must always submit the + * previously allocated bio for IO before attempting to allocate a new one. + * Failure to do so can cause deadlocks under memory pressure. * - * Note that when running under submit_bio_noacct() (i.e. any block - * driver), bios are not submitted until after you return - see the code in - * submit_bio_noacct() that converts recursion into iteration, to prevent - * stack overflows. + * Note that when running under submit_bio_noacct() (i.e. any block driver), + * bios are not submitted until after you return - see the code in + * submit_bio_noacct() that converts recursion into iteration, to prevent + * stack overflows. * - * This would normally mean allocating multiple bios under - * submit_bio_noacct() would be susceptible to deadlocks, but we have - * deadlock avoidance code that resubmits any blocked bios from a rescuer - * thread. + * This would normally mean allocating multiple bios under submit_bio_noacct() + * would be susceptible to deadlocks, but we have + * deadlock avoidance code that resubmits any blocked bios from a rescuer + * thread. * - * However, we do not guarantee forward progress for allocations from other - * mempools. Doing multiple allocations from the same mempool under - * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad - * for per bio allocations. + * However, we do not guarantee forward progress for allocations from other + * mempools. Doing multiple allocations from the same mempool under + * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad + * for per bio allocations. * - * RETURNS: - * Pointer to new bio on success, NULL on failure. + * Returns: Pointer to new bio on success, NULL on failure. */ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs, struct bio_set *bs) { gfp_t saved_gfp = gfp_mask; - unsigned front_pad; - unsigned inline_vecs; - struct bio_vec *bvl = NULL; struct bio *bio; void *p; - if (!bs) { - if (nr_iovecs > UIO_MAXIOV) - return NULL; - - p = kmalloc(struct_size(bio, bi_inline_vecs, nr_iovecs), gfp_mask); - front_pad = 0; - inline_vecs = nr_iovecs; - } else { - /* should not use nobvec bioset for nr_iovecs > 0 */ - if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && - nr_iovecs > 0)) - return NULL; - /* - * submit_bio_noacct() converts recursion to iteration; this - * means if we're running beneath it, any bios we allocate and - * submit will not be submitted (and thus freed) until after we - * return. - * - * This exposes us to a potential deadlock if we allocate - * multiple bios from the same bio_set() while running - * underneath submit_bio_noacct(). If we were to allocate - * multiple bios (say a stacking block driver that was splitting - * bios), we would deadlock if we exhausted the mempool's - * reserve. - * - * We solve this, and guarantee forward progress, with a rescuer - * workqueue per bio_set. If we go to allocate and there are - * bios on current->bio_list, we first try the allocation - * without __GFP_DIRECT_RECLAIM; if that fails, we punt those - * bios we would be blocking to the rescuer workqueue before - * we retry with the original gfp_flags. - */ - - if (current->bio_list && - (!bio_list_empty(¤t->bio_list[0]) || - !bio_list_empty(¤t->bio_list[1])) && - bs->rescue_workqueue) - gfp_mask &= ~__GFP_DIRECT_RECLAIM; + /* should not use nobvec bioset for nr_iovecs > 0 */ + if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_iovecs > 0)) + return NULL; + /* + * submit_bio_noacct() converts recursion to iteration; this means if + * we're running beneath it, any bios we allocate and submit will not be + * submitted (and thus freed) until after we return. + * + * This exposes us to a potential deadlock if we allocate multiple bios + * from the same bio_set() while running underneath submit_bio_noacct(). + * If we were to allocate multiple bios (say a stacking block driver + * that was splitting bios), we would deadlock if we exhausted the + * mempool's reserve. + * + * We solve this, and guarantee forward progress, with a rescuer + * workqueue per bio_set. If we go to allocate and there are bios on + * current->bio_list, we first try the allocation without + * __GFP_DIRECT_RECLAIM; if that fails, we punt those bios we would be + * blocking to the rescuer workqueue before we retry with the original + * gfp_flags. + */ + if (current->bio_list && + (!bio_list_empty(¤t->bio_list[0]) || + !bio_list_empty(¤t->bio_list[1])) && + bs->rescue_workqueue) + gfp_mask &= ~__GFP_DIRECT_RECLAIM; + + p = mempool_alloc(&bs->bio_pool, gfp_mask); + if (!p && gfp_mask != saved_gfp) { + punt_bios_to_rescuer(bs); + gfp_mask = saved_gfp; p = mempool_alloc(&bs->bio_pool, gfp_mask); - if (!p && gfp_mask != saved_gfp) { - punt_bios_to_rescuer(bs); - gfp_mask = saved_gfp; - p = mempool_alloc(&bs->bio_pool, gfp_mask); - } - - front_pad = bs->front_pad; - inline_vecs = BIO_INLINE_VECS; } - if (unlikely(!p)) return NULL; - bio = p + front_pad; - bio_init(bio, NULL, 0); - - if (nr_iovecs > inline_vecs) { + bio = p + bs->front_pad; + if (nr_iovecs > BIO_INLINE_VECS) { unsigned long idx = 0; + struct bio_vec *bvl = NULL; bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool); if (!bvl && gfp_mask != saved_gfp) { punt_bios_to_rescuer(bs); gfp_mask = saved_gfp; - bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool); + bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, + &bs->bvec_pool); } if (unlikely(!bvl)) goto err_free; bio->bi_flags |= idx << BVEC_POOL_OFFSET; - bio->bi_max_vecs = bvec_nr_vecs(idx); + bio_init(bio, bvl, bvec_nr_vecs(idx)); } else if (nr_iovecs) { - bvl = bio->bi_inline_vecs; - bio->bi_max_vecs = inline_vecs; + bio_init(bio, bio->bi_inline_vecs, BIO_INLINE_VECS); + } else { + bio_init(bio, NULL, 0); } bio->bi_pool = bs; - bio->bi_io_vec = bvl; return bio; err_free: @@ -521,6 +499,31 @@ err_free: } EXPORT_SYMBOL(bio_alloc_bioset); +/** + * bio_kmalloc - kmalloc a bio for I/O + * @gfp_mask: the GFP_* mask given to the slab allocator + * @nr_iovecs: number of iovecs to pre-allocate + * + * Use kmalloc to allocate and initialize a bio. + * + * Returns: Pointer to new bio on success, NULL on failure. + */ +struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs) +{ + struct bio *bio; + + if (nr_iovecs > UIO_MAXIOV) + return NULL; + + bio = kmalloc(struct_size(bio, bi_inline_vecs, nr_iovecs), gfp_mask); + if (unlikely(!bio)) + return NULL; + bio_init(bio, nr_iovecs ? bio->bi_inline_vecs : NULL, nr_iovecs); + bio->bi_pool = NULL; + return bio; +} +EXPORT_SYMBOL(bio_kmalloc); + void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start) { unsigned long flags; diff --git a/include/linux/bio.h b/include/linux/bio.h index 676870b2c88d..c74857cf1252 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -408,6 +408,7 @@ extern int biovec_init_pool(mempool_t *pool, int pool_entries); extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src); extern struct bio *bio_alloc_bioset(gfp_t, unsigned int, struct bio_set *); +struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs); extern void bio_put(struct bio *); extern void __bio_clone_fast(struct bio *, struct bio *); @@ -420,11 +421,6 @@ static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs) return bio_alloc_bioset(gfp_mask, nr_iovecs, &fs_bio_set); } -static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs) -{ - return bio_alloc_bioset(gfp_mask, nr_iovecs, NULL); -} - extern blk_qc_t submit_bio(struct bio *); extern void bio_endio(struct bio *); -- cgit v1.2.3 From c6bf3f0e25f4c0f0ecce6cf8d1c589bd9d74d3cf Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 26 Jan 2021 15:52:35 +0100 Subject: block: use an on-stack bio in blkdev_issue_flush There is no point in allocating memory for a synchronous flush. Signed-off-by: Christoph Hellwig Reviewed-by: Johannes Thumshirn Reviewed-by: Chaitanya Kulkarni Acked-by: Damien Le Moal Signed-off-by: Jens Axboe --- block/blk-flush.c | 17 ++++++----------- drivers/md/dm-zoned-metadata.c | 6 +++--- drivers/md/raid5-ppl.c | 2 +- drivers/nvme/target/io-cmd-bdev.c | 2 +- fs/block_dev.c | 2 +- fs/exfat/file.c | 2 +- fs/ext4/fast_commit.c | 4 ++-- fs/ext4/fsync.c | 2 +- fs/ext4/ialloc.c | 2 +- fs/ext4/super.c | 2 +- fs/fat/file.c | 2 +- fs/hfsplus/inode.c | 2 +- fs/hfsplus/super.c | 2 +- fs/jbd2/checkpoint.c | 2 +- fs/jbd2/commit.c | 4 ++-- fs/jbd2/recovery.c | 2 +- fs/libfs.c | 2 +- fs/nilfs2/the_nilfs.h | 2 +- fs/ocfs2/file.c | 2 +- fs/reiserfs/file.c | 2 +- fs/xfs/xfs_super.c | 2 +- fs/zonefs/super.c | 2 +- include/linux/blkdev.h | 4 ++-- 23 files changed, 33 insertions(+), 38 deletions(-) (limited to 'include/linux') diff --git a/block/blk-flush.c b/block/blk-flush.c index 76c1624cb06c..7942ca6ed321 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -432,23 +432,18 @@ void blk_insert_flush(struct request *rq) /** * blkdev_issue_flush - queue a flush * @bdev: blockdev to issue flush for - * @gfp_mask: memory allocation flags (for bio_alloc) * * Description: * Issue a flush for the block device in question. */ -int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask) +int blkdev_issue_flush(struct block_device *bdev) { - struct bio *bio; - int ret = 0; + struct bio bio; - bio = bio_alloc(gfp_mask, 0); - bio_set_dev(bio, bdev); - bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; - - ret = submit_bio_wait(bio); - bio_put(bio); - return ret; + bio_init(&bio, NULL, 0); + bio_set_dev(&bio, bdev); + bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; + return submit_bio_wait(&bio); } EXPORT_SYMBOL(blkdev_issue_flush); diff --git a/drivers/md/dm-zoned-metadata.c b/drivers/md/dm-zoned-metadata.c index b298fefb022e..039d17b28938 100644 --- a/drivers/md/dm-zoned-metadata.c +++ b/drivers/md/dm-zoned-metadata.c @@ -819,7 +819,7 @@ static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set) ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block, mblk->page); if (ret == 0) - ret = blkdev_issue_flush(dev->bdev, GFP_NOIO); + ret = blkdev_issue_flush(dev->bdev); return ret; } @@ -862,7 +862,7 @@ static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd, /* Flush drive cache (this will also sync data) */ if (ret == 0) - ret = blkdev_issue_flush(dev->bdev, GFP_NOIO); + ret = blkdev_issue_flush(dev->bdev); return ret; } @@ -933,7 +933,7 @@ int dmz_flush_metadata(struct dmz_metadata *zmd) /* If there are no dirty metadata blocks, just flush the device cache */ if (list_empty(&write_list)) { - ret = blkdev_issue_flush(dev->bdev, GFP_NOIO); + ret = blkdev_issue_flush(dev->bdev); goto err; } diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c index d0f540296fe9..e8c118e05dfd 100644 --- a/drivers/md/raid5-ppl.c +++ b/drivers/md/raid5-ppl.c @@ -1037,7 +1037,7 @@ static int ppl_recover(struct ppl_log *log, struct ppl_header *pplhdr, } /* flush the disk cache after recovery if necessary */ - ret = blkdev_issue_flush(rdev->bdev, GFP_KERNEL); + ret = blkdev_issue_flush(rdev->bdev); out: __free_page(page); return ret; diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index 125dde3f410e..bf6e0ac9ad28 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -333,7 +333,7 @@ static void nvmet_bdev_execute_flush(struct nvmet_req *req) u16 nvmet_bdev_flush(struct nvmet_req *req) { - if (blkdev_issue_flush(req->ns->bdev, GFP_KERNEL)) + if (blkdev_issue_flush(req->ns->bdev)) return NVME_SC_INTERNAL | NVME_SC_DNR; return 0; } diff --git a/fs/block_dev.c b/fs/block_dev.c index c1fe29dac485..9d4b1a884d76 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -680,7 +680,7 @@ int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync) * i_mutex and doing so causes performance issues with concurrent * O_SYNC writers to a block device. */ - error = blkdev_issue_flush(bdev, GFP_KERNEL); + error = blkdev_issue_flush(bdev); if (error == -EOPNOTSUPP) error = 0; diff --git a/fs/exfat/file.c b/fs/exfat/file.c index a92478eabfa4..183ffdf4d43c 100644 --- a/fs/exfat/file.c +++ b/fs/exfat/file.c @@ -361,7 +361,7 @@ int exfat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) if (err) return err; - return blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + return blkdev_issue_flush(inode->i_sb->s_bdev); } const struct file_operations exfat_file_operations = { diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index 0a14a7c87bf8..6e8208acfc62 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -1076,7 +1076,7 @@ static int ext4_fc_perform_commit(journal_t *journal) * flush before we start writing fast commit blocks. */ if (journal->j_fs_dev != journal->j_dev) - blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS); + blkdev_issue_flush(journal->j_fs_dev); blk_start_plug(&plug); if (sbi->s_fc_bytes == 0) { @@ -1535,7 +1535,7 @@ static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl) out: iput(inode); if (!ret) - blkdev_issue_flush(sb->s_bdev, GFP_KERNEL); + blkdev_issue_flush(sb->s_bdev); return 0; } diff --git a/fs/ext4/fsync.c b/fs/ext4/fsync.c index 113bfb023a4a..027a7d7037a0 100644 --- a/fs/ext4/fsync.c +++ b/fs/ext4/fsync.c @@ -174,7 +174,7 @@ int ext4_sync_file(struct file *file, loff_t start, loff_t end, int datasync) ret = ext4_fsync_journal(inode, datasync, &needs_barrier); if (needs_barrier) { - err = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + err = blkdev_issue_flush(inode->i_sb->s_bdev); if (!ret) ret = err; } diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index b215c564bc31..20f2fcb799f5 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -1583,7 +1583,7 @@ int ext4_init_inode_table(struct super_block *sb, ext4_group_t group, if (ret < 0) goto err_out; if (barrier) - blkdev_issue_flush(sb->s_bdev, GFP_NOFS); + blkdev_issue_flush(sb->s_bdev); skip_zeroout: ext4_lock_group(sb, group); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 9a6f9875aa34..fb5985102c1d 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -5709,7 +5709,7 @@ static int ext4_sync_fs(struct super_block *sb, int wait) needs_barrier = true; if (needs_barrier) { int err; - err = blkdev_issue_flush(sb->s_bdev, GFP_KERNEL); + err = blkdev_issue_flush(sb->s_bdev); if (!ret) ret = err; } diff --git a/fs/fat/file.c b/fs/fat/file.c index f9ee27cf4d7c..5fee74f1ad61 100644 --- a/fs/fat/file.c +++ b/fs/fat/file.c @@ -195,7 +195,7 @@ int fat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) if (err) return err; - return blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + return blkdev_issue_flush(inode->i_sb->s_bdev); } diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index e3da9e96b835..ca464328b79c 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -340,7 +340,7 @@ int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end, } if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags)) - blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + blkdev_issue_flush(inode->i_sb->s_bdev); inode_unlock(inode); diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c index 807119ae5adf..b9e3db3f855f 100644 --- a/fs/hfsplus/super.c +++ b/fs/hfsplus/super.c @@ -239,7 +239,7 @@ out: mutex_unlock(&sbi->vh_mutex); if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags)) - blkdev_issue_flush(sb->s_bdev, GFP_KERNEL); + blkdev_issue_flush(sb->s_bdev); return error; } diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c index 472932b9e6bc..63b526d44886 100644 --- a/fs/jbd2/checkpoint.c +++ b/fs/jbd2/checkpoint.c @@ -416,7 +416,7 @@ int jbd2_cleanup_journal_tail(journal_t *journal) * jbd2_cleanup_journal_tail() doesn't get called all that often. */ if (journal->j_flags & JBD2_BARRIER) - blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS); + blkdev_issue_flush(journal->j_fs_dev); return __jbd2_update_log_tail(journal, first_tid, blocknr); } diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c index b121d7d434c6..3cc4ab2ba7f4 100644 --- a/fs/jbd2/commit.c +++ b/fs/jbd2/commit.c @@ -825,7 +825,7 @@ start_journal_io: if (commit_transaction->t_need_data_flush && (journal->j_fs_dev != journal->j_dev) && (journal->j_flags & JBD2_BARRIER)) - blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS); + blkdev_issue_flush(journal->j_fs_dev); /* Done it all: now write the commit record asynchronously. */ if (jbd2_has_feature_async_commit(journal)) { @@ -932,7 +932,7 @@ start_journal_io: stats.run.rs_blocks_logged++; if (jbd2_has_feature_async_commit(journal) && journal->j_flags & JBD2_BARRIER) { - blkdev_issue_flush(journal->j_dev, GFP_NOFS); + blkdev_issue_flush(journal->j_dev); } if (err) diff --git a/fs/jbd2/recovery.c b/fs/jbd2/recovery.c index dc0694fcfcd1..69f18fe20923 100644 --- a/fs/jbd2/recovery.c +++ b/fs/jbd2/recovery.c @@ -326,7 +326,7 @@ int jbd2_journal_recover(journal_t *journal) err = err2; /* Make sure all replayed data is on permanent storage */ if (journal->j_flags & JBD2_BARRIER) { - err2 = blkdev_issue_flush(journal->j_fs_dev, GFP_KERNEL); + err2 = blkdev_issue_flush(journal->j_fs_dev); if (!err) err = err2; } diff --git a/fs/libfs.c b/fs/libfs.c index d1c3bade9f30..8398a0efb401 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -1117,7 +1117,7 @@ int generic_file_fsync(struct file *file, loff_t start, loff_t end, err = __generic_file_fsync(file, start, end, datasync); if (err) return err; - return blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + return blkdev_issue_flush(inode->i_sb->s_bdev); } EXPORT_SYMBOL(generic_file_fsync); diff --git a/fs/nilfs2/the_nilfs.h b/fs/nilfs2/the_nilfs.h index b55cdeb4d169..987c8ab02aee 100644 --- a/fs/nilfs2/the_nilfs.h +++ b/fs/nilfs2/the_nilfs.h @@ -375,7 +375,7 @@ static inline int nilfs_flush_device(struct the_nilfs *nilfs) */ smp_wmb(); - err = blkdev_issue_flush(nilfs->ns_bdev, GFP_KERNEL); + err = blkdev_issue_flush(nilfs->ns_bdev); if (err != -EIO) err = 0; return err; diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 85979e2214b3..df6d709d2ae3 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -194,7 +194,7 @@ static int ocfs2_sync_file(struct file *file, loff_t start, loff_t end, needs_barrier = true; err = jbd2_complete_transaction(journal, commit_tid); if (needs_barrier) { - ret = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + ret = blkdev_issue_flush(inode->i_sb->s_bdev); if (!err) err = ret; } diff --git a/fs/reiserfs/file.c b/fs/reiserfs/file.c index 0b641ae694f1..1db0254bc38b 100644 --- a/fs/reiserfs/file.c +++ b/fs/reiserfs/file.c @@ -159,7 +159,7 @@ static int reiserfs_sync_file(struct file *filp, loff_t start, loff_t end, barrier_done = reiserfs_commit_for_inode(inode); reiserfs_write_unlock(inode->i_sb); if (barrier_done != 1 && reiserfs_barrier_flush(inode->i_sb)) - blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + blkdev_issue_flush(inode->i_sb->s_bdev); inode_unlock(inode); if (barrier_done < 0) return barrier_done; diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 813be879a5e5..c3e32789829f 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -342,7 +342,7 @@ void xfs_blkdev_issue_flush( xfs_buftarg_t *buftarg) { - blkdev_issue_flush(buftarg->bt_bdev, GFP_NOFS); + blkdev_issue_flush(buftarg->bt_bdev); } STATIC void diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c index faea2ed34b4a..ab68e27bb322 100644 --- a/fs/zonefs/super.c +++ b/fs/zonefs/super.c @@ -541,7 +541,7 @@ static int zonefs_file_fsync(struct file *file, loff_t start, loff_t end, if (ZONEFS_I(inode)->i_ztype == ZONEFS_ZTYPE_CNV) ret = file_write_and_wait_range(file, start, end); if (!ret) - ret = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL); + ret = blkdev_issue_flush(inode->i_sb->s_bdev); if (ret) zonefs_io_error(inode, true); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 2491e17b61c4..0dea268bd61b 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1288,7 +1288,7 @@ static inline bool blk_needs_flush_plug(struct task_struct *tsk) !list_empty(&plug->cb_list)); } -int blkdev_issue_flush(struct block_device *, gfp_t); +int blkdev_issue_flush(struct block_device *bdev); long nr_blockdev_pages(void); #else /* CONFIG_BLOCK */ struct blk_plug { @@ -1316,7 +1316,7 @@ static inline bool blk_needs_flush_plug(struct task_struct *tsk) return false; } -static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask) +static inline int blkdev_issue_flush(struct block_device *bdev) { return 0; } -- cgit v1.2.3 From 48d15436fde6feebcded7bd0fdc8ea4a9181b8fa Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 26 Jan 2021 15:52:47 +0100 Subject: mm: remove get_swap_bio Just reuse the block_device and sector from the swap_info structure, just as used by the SWP_SYNCHRONOUS path. Also remove the checks for NULL returns from bio_alloc as that can't happen for sleeping allocations. Signed-off-by: Christoph Hellwig Reviewed-by: Johannes Thumshirn Reviewed-by: Chaitanya Kulkarni Acked-by: Damien Le Moal Signed-off-by: Jens Axboe --- include/linux/swap.h | 1 - mm/page_io.c | 45 +++++++++++++-------------------------------- mm/swapfile.c | 10 ---------- 3 files changed, 13 insertions(+), 43 deletions(-) (limited to 'include/linux') diff --git a/include/linux/swap.h b/include/linux/swap.h index 596bc2f4d9b0..3f1f7ae0fbe9 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -468,7 +468,6 @@ extern int free_swap_and_cache(swp_entry_t); int swap_type_of(dev_t device, sector_t offset); int find_first_swap(dev_t *device); extern unsigned int count_swap_pages(int, int); -extern sector_t map_swap_page(struct page *, struct block_device **); extern sector_t swapdev_block(int, pgoff_t); extern int page_swapcount(struct page *); extern int __swap_count(swp_entry_t entry); diff --git a/mm/page_io.c b/mm/page_io.c index a75f35464a4e..92f7941c6d01 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -26,25 +26,6 @@ #include #include -static struct bio *get_swap_bio(gfp_t gfp_flags, - struct page *page, bio_end_io_t end_io) -{ - struct bio *bio; - - bio = bio_alloc(gfp_flags, 1); - if (bio) { - struct block_device *bdev; - - bio->bi_iter.bi_sector = map_swap_page(page, &bdev); - bio_set_dev(bio, bdev); - bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9; - bio->bi_end_io = end_io; - - bio_add_page(bio, page, thp_size(page), 0); - } - return bio; -} - void end_swap_bio_write(struct bio *bio) { struct page *page = bio_first_page_all(bio); @@ -361,13 +342,13 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc, return 0; } - bio = get_swap_bio(GFP_NOIO, page, end_write_func); - if (bio == NULL) { - set_page_dirty(page); - unlock_page(page); - return -ENOMEM; - } + bio = bio_alloc(GFP_NOIO, 1); + bio_set_dev(bio, sis->bdev); + bio->bi_iter.bi_sector = swap_page_sector(page); bio->bi_opf = REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc); + bio->bi_end_io = end_write_func; + bio_add_page(bio, page, thp_size(page), 0); + bio_associate_blkg_from_page(bio, page); count_swpout_vm_event(page); set_page_writeback(page); @@ -427,18 +408,18 @@ int swap_readpage(struct page *page, bool synchronous) } ret = 0; - bio = get_swap_bio(GFP_KERNEL, page, end_swap_bio_read); - if (bio == NULL) { - unlock_page(page); - ret = -ENOMEM; - goto out; - } + bio = bio_alloc(GFP_KERNEL, 1); + bio_set_dev(bio, sis->bdev); + bio->bi_opf = REQ_OP_READ; + bio->bi_iter.bi_sector = swap_page_sector(page); + bio->bi_end_io = end_swap_bio_read; + bio_add_page(bio, page, thp_size(page), 0); + disk = bio->bi_bdev->bd_disk; /* * Keep this task valid during swap readpage because the oom killer may * attempt to access it in the page fault retry time check. */ - bio_set_op_attrs(bio, REQ_OP_READ, 0); if (synchronous) { bio->bi_opf |= REQ_HIPRI; get_task_struct(current); diff --git a/mm/swapfile.c b/mm/swapfile.c index 9fffc5af29d1..bfa9e8b0c2ef 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2301,16 +2301,6 @@ static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev) return se->start_block + (offset - se->start_page); } -/* - * Returns the page offset into bdev for the specified page's swap entry. - */ -sector_t map_swap_page(struct page *page, struct block_device **bdev) -{ - swp_entry_t entry; - entry.val = page_private(page); - return map_swap_entry(entry, bdev); -} - /* * Free all of a swapdev's extent information */ -- cgit v1.2.3 From 84f9017c37c479c4f70456a645d24d2296ad2208 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Mon, 25 Jan 2021 12:59:57 +0100 Subject: ACPI: platform-profile: Introduce object pointers to callbacks Add an object pointer to handler callbacks to avoid the need for drivers to have a global variable to get to their driver-data struct. Link: https://lore.kernel.org/linux-acpi/6a29f338-d9e4-150c-81dd-2ffb54f5bc35@redhat.com/ Link: https://lore.kernel.org/r/20210114073429.176462-3-jiaxun.yang@flygoat.com Signed-off-by: Jiaxun Yang Suggested-by: Hans de Goede Signed-off-by: Hans de Goede Signed-off-by: Rafael J. Wysocki --- drivers/acpi/platform_profile.c | 4 ++-- include/linux/platform_profile.h | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/platform_profile.c b/drivers/acpi/platform_profile.c index f65c61db7921..80e9df427eb8 100644 --- a/drivers/acpi/platform_profile.c +++ b/drivers/acpi/platform_profile.c @@ -64,7 +64,7 @@ static ssize_t platform_profile_show(struct device *dev, return -ENODEV; } - err = cur_profile->profile_get(&profile); + err = cur_profile->profile_get(cur_profile, &profile); mutex_unlock(&profile_lock); if (err) return err; @@ -104,7 +104,7 @@ static ssize_t platform_profile_store(struct device *dev, return -EOPNOTSUPP; } - err = cur_profile->profile_set(i); + err = cur_profile->profile_set(cur_profile, i); mutex_unlock(&profile_lock); if (err) return err; diff --git a/include/linux/platform_profile.h b/include/linux/platform_profile.h index c797fdb3d91a..a26542d53058 100644 --- a/include/linux/platform_profile.h +++ b/include/linux/platform_profile.h @@ -28,8 +28,10 @@ enum platform_profile_option { struct platform_profile_handler { unsigned long choices[BITS_TO_LONGS(PLATFORM_PROFILE_LAST)]; - int (*profile_get)(enum platform_profile_option *profile); - int (*profile_set)(enum platform_profile_option profile); + int (*profile_get)(struct platform_profile_handler *pprof, + enum platform_profile_option *profile); + int (*profile_set)(struct platform_profile_handler *pprof, + enum platform_profile_option profile); }; int platform_profile_register(struct platform_profile_handler *pprof); -- cgit v1.2.3 From 0bfa0820c274b019583b3454c6c889c99c24558d Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Mon, 25 Jan 2021 14:29:18 -0500 Subject: PM: clk: make PM clock layer compatible with clocks that must sleep The clock API splits its interface into sleepable ant atomic contexts: - clk_prepare/clk_unprepare for stuff that might sleep - clk_enable_clk_disable for anything that may be done in atomic context The code handling runtime PM for clocks only calls clk_disable() on suspend requests, and clk_enable on resume requests. This means that runtime PM with clock providers that only have the prepare/unprepare methods implemented is basically useless. Many clock implementations can't accommodate atomic contexts. This is often the case when communication with the clock happens through another subsystem like I2C or SCMI. Let's make the clock PM code useful with such clocks by safely invoking clk_prepare/clk_unprepare upon resume/suspend requests. Of course, when such clocks are registered with the PM layer then pm_runtime_irq_safe() can't be used, and neither pm_runtime_suspend() nor pm_runtime_resume() may be invoked in atomic context. For clocks that do implement the enable and disable methods then everything just works as before. A note on sparse: According to https://lwn.net/Articles/109066/ there are things that sparse can't cope with. In particular, pm_clk_op_lock() and pm_clk_op_unlock() may or may not lock/unlock psd->lock depending on some runtime condition. To work around that we tell it the lock is always untaken for the purpose of static analisys. Thanks to Naresh Kamboju for reporting issues with the initial patch. Signed-off-by: Nicolas Pitre Tested-by: Naresh Kamboju Signed-off-by: Rafael J. Wysocki --- drivers/base/power/clock_ops.c | 223 +++++++++++++++++++++++++++++++++-------- drivers/clk/clk.c | 21 ++++ include/linux/clk.h | 24 ++++- include/linux/pm.h | 2 + 4 files changed, 228 insertions(+), 42 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/power/clock_ops.c b/drivers/base/power/clock_ops.c index ced6863a16a5..84d5acb6301b 100644 --- a/drivers/base/power/clock_ops.c +++ b/drivers/base/power/clock_ops.c @@ -23,6 +23,7 @@ enum pce_status { PCE_STATUS_NONE = 0, PCE_STATUS_ACQUIRED, + PCE_STATUS_PREPARED, PCE_STATUS_ENABLED, PCE_STATUS_ERROR, }; @@ -32,8 +33,112 @@ struct pm_clock_entry { char *con_id; struct clk *clk; enum pce_status status; + bool enabled_when_prepared; }; +/** + * pm_clk_list_lock - ensure exclusive access for modifying the PM clock + * entry list. + * @psd: pm_subsys_data instance corresponding to the PM clock entry list + * and clk_op_might_sleep count to be modified. + * + * Get exclusive access before modifying the PM clock entry list and the + * clock_op_might_sleep count to guard against concurrent modifications. + * This also protects against a concurrent clock_op_might_sleep and PM clock + * entry list usage in pm_clk_suspend()/pm_clk_resume() that may or may not + * happen in atomic context, hence both the mutex and the spinlock must be + * taken here. + */ +static void pm_clk_list_lock(struct pm_subsys_data *psd) + __acquires(&psd->lock) +{ + mutex_lock(&psd->clock_mutex); + spin_lock_irq(&psd->lock); +} + +/** + * pm_clk_list_unlock - counterpart to pm_clk_list_lock(). + * @psd: the same pm_subsys_data instance previously passed to + * pm_clk_list_lock(). + */ +static void pm_clk_list_unlock(struct pm_subsys_data *psd) + __releases(&psd->lock) +{ + spin_unlock_irq(&psd->lock); + mutex_unlock(&psd->clock_mutex); +} + +/** + * pm_clk_op_lock - ensure exclusive access for performing clock operations. + * @psd: pm_subsys_data instance corresponding to the PM clock entry list + * and clk_op_might_sleep count being used. + * @flags: stored irq flags. + * @fn: string for the caller function's name. + * + * This is used by pm_clk_suspend() and pm_clk_resume() to guard + * against concurrent modifications to the clock entry list and the + * clock_op_might_sleep count. If clock_op_might_sleep is != 0 then + * only the mutex can be locked and those functions can only be used in + * non atomic context. If clock_op_might_sleep == 0 then these functions + * may be used in any context and only the spinlock can be locked. + * Returns -EINVAL if called in atomic context when clock ops might sleep. + */ +static int pm_clk_op_lock(struct pm_subsys_data *psd, unsigned long *flags, + const char *fn) + /* sparse annotations don't work here as exit state isn't static */ +{ + bool atomic_context = in_atomic() || irqs_disabled(); + +try_again: + spin_lock_irqsave(&psd->lock, *flags); + if (!psd->clock_op_might_sleep) { + /* the __release is there to work around sparse limitations */ + __release(&psd->lock); + return 0; + } + + /* bail out if in atomic context */ + if (atomic_context) { + pr_err("%s: atomic context with clock_ops_might_sleep = %d", + fn, psd->clock_op_might_sleep); + spin_unlock_irqrestore(&psd->lock, *flags); + might_sleep(); + return -EPERM; + } + + /* we must switch to the mutex */ + spin_unlock_irqrestore(&psd->lock, *flags); + mutex_lock(&psd->clock_mutex); + + /* + * There was a possibility for psd->clock_op_might_sleep + * to become 0 above. Keep the mutex only if not the case. + */ + if (likely(psd->clock_op_might_sleep)) + return 0; + + mutex_unlock(&psd->clock_mutex); + goto try_again; +} + +/** + * pm_clk_op_unlock - counterpart to pm_clk_op_lock(). + * @psd: the same pm_subsys_data instance previously passed to + * pm_clk_op_lock(). + * @flags: irq flags provided by pm_clk_op_lock(). + */ +static void pm_clk_op_unlock(struct pm_subsys_data *psd, unsigned long *flags) + /* sparse annotations don't work here as entry state isn't static */ +{ + if (psd->clock_op_might_sleep) { + mutex_unlock(&psd->clock_mutex); + } else { + /* the __acquire is there to work around sparse limitations */ + __acquire(&psd->lock); + spin_unlock_irqrestore(&psd->lock, *flags); + } +} + /** * pm_clk_enable - Enable a clock, reporting any errors * @dev: The device for the given clock @@ -43,14 +148,21 @@ static inline void __pm_clk_enable(struct device *dev, struct pm_clock_entry *ce { int ret; - if (ce->status < PCE_STATUS_ERROR) { + switch (ce->status) { + case PCE_STATUS_ACQUIRED: + ret = clk_prepare_enable(ce->clk); + break; + case PCE_STATUS_PREPARED: ret = clk_enable(ce->clk); - if (!ret) - ce->status = PCE_STATUS_ENABLED; - else - dev_err(dev, "%s: failed to enable clk %p, error %d\n", - __func__, ce->clk, ret); + break; + default: + return; } + if (!ret) + ce->status = PCE_STATUS_ENABLED; + else + dev_err(dev, "%s: failed to enable clk %p, error %d\n", + __func__, ce->clk, ret); } /** @@ -64,17 +176,20 @@ static void pm_clk_acquire(struct device *dev, struct pm_clock_entry *ce) ce->clk = clk_get(dev, ce->con_id); if (IS_ERR(ce->clk)) { ce->status = PCE_STATUS_ERROR; + return; + } else if (clk_is_enabled_when_prepared(ce->clk)) { + /* we defer preparing the clock in that case */ + ce->status = PCE_STATUS_ACQUIRED; + ce->enabled_when_prepared = true; + } else if (clk_prepare(ce->clk)) { + ce->status = PCE_STATUS_ERROR; + dev_err(dev, "clk_prepare() failed\n"); + return; } else { - if (clk_prepare(ce->clk)) { - ce->status = PCE_STATUS_ERROR; - dev_err(dev, "clk_prepare() failed\n"); - } else { - ce->status = PCE_STATUS_ACQUIRED; - dev_dbg(dev, - "Clock %pC con_id %s managed by runtime PM.\n", - ce->clk, ce->con_id); - } + ce->status = PCE_STATUS_PREPARED; } + dev_dbg(dev, "Clock %pC con_id %s managed by runtime PM.\n", + ce->clk, ce->con_id); } static int __pm_clk_add(struct device *dev, const char *con_id, @@ -106,9 +221,11 @@ static int __pm_clk_add(struct device *dev, const char *con_id, pm_clk_acquire(dev, ce); - spin_lock_irq(&psd->lock); + pm_clk_list_lock(psd); list_add_tail(&ce->node, &psd->clock_list); - spin_unlock_irq(&psd->lock); + if (ce->enabled_when_prepared) + psd->clock_op_might_sleep++; + pm_clk_list_unlock(psd); return 0; } @@ -239,14 +356,20 @@ static void __pm_clk_remove(struct pm_clock_entry *ce) if (!ce) return; - if (ce->status < PCE_STATUS_ERROR) { - if (ce->status == PCE_STATUS_ENABLED) - clk_disable(ce->clk); - - if (ce->status >= PCE_STATUS_ACQUIRED) { - clk_unprepare(ce->clk); + switch (ce->status) { + case PCE_STATUS_ENABLED: + clk_disable(ce->clk); + fallthrough; + case PCE_STATUS_PREPARED: + clk_unprepare(ce->clk); + fallthrough; + case PCE_STATUS_ACQUIRED: + case PCE_STATUS_ERROR: + if (!IS_ERR(ce->clk)) clk_put(ce->clk); - } + break; + default: + break; } kfree(ce->con_id); @@ -269,7 +392,7 @@ void pm_clk_remove(struct device *dev, const char *con_id) if (!psd) return; - spin_lock_irq(&psd->lock); + pm_clk_list_lock(psd); list_for_each_entry(ce, &psd->clock_list, node) { if (!con_id && !ce->con_id) @@ -280,12 +403,14 @@ void pm_clk_remove(struct device *dev, const char *con_id) goto remove; } - spin_unlock_irq(&psd->lock); + pm_clk_list_unlock(psd); return; remove: list_del(&ce->node); - spin_unlock_irq(&psd->lock); + if (ce->enabled_when_prepared) + psd->clock_op_might_sleep--; + pm_clk_list_unlock(psd); __pm_clk_remove(ce); } @@ -307,19 +432,21 @@ void pm_clk_remove_clk(struct device *dev, struct clk *clk) if (!psd || !clk) return; - spin_lock_irq(&psd->lock); + pm_clk_list_lock(psd); list_for_each_entry(ce, &psd->clock_list, node) { if (clk == ce->clk) goto remove; } - spin_unlock_irq(&psd->lock); + pm_clk_list_unlock(psd); return; remove: list_del(&ce->node); - spin_unlock_irq(&psd->lock); + if (ce->enabled_when_prepared) + psd->clock_op_might_sleep--; + pm_clk_list_unlock(psd); __pm_clk_remove(ce); } @@ -330,13 +457,16 @@ EXPORT_SYMBOL_GPL(pm_clk_remove_clk); * @dev: Device to initialize the list of PM clocks for. * * Initialize the lock and clock_list members of the device's pm_subsys_data - * object. + * object, set the count of clocks that might sleep to 0. */ void pm_clk_init(struct device *dev) { struct pm_subsys_data *psd = dev_to_psd(dev); - if (psd) + if (psd) { INIT_LIST_HEAD(&psd->clock_list); + mutex_init(&psd->clock_mutex); + psd->clock_op_might_sleep = 0; + } } EXPORT_SYMBOL_GPL(pm_clk_init); @@ -372,12 +502,13 @@ void pm_clk_destroy(struct device *dev) INIT_LIST_HEAD(&list); - spin_lock_irq(&psd->lock); + pm_clk_list_lock(psd); list_for_each_entry_safe_reverse(ce, c, &psd->clock_list, node) list_move(&ce->node, &list); + psd->clock_op_might_sleep = 0; - spin_unlock_irq(&psd->lock); + pm_clk_list_unlock(psd); dev_pm_put_subsys_data(dev); @@ -397,23 +528,30 @@ int pm_clk_suspend(struct device *dev) struct pm_subsys_data *psd = dev_to_psd(dev); struct pm_clock_entry *ce; unsigned long flags; + int ret; dev_dbg(dev, "%s()\n", __func__); if (!psd) return 0; - spin_lock_irqsave(&psd->lock, flags); + ret = pm_clk_op_lock(psd, &flags, __func__); + if (ret) + return ret; list_for_each_entry_reverse(ce, &psd->clock_list, node) { - if (ce->status < PCE_STATUS_ERROR) { - if (ce->status == PCE_STATUS_ENABLED) + if (ce->status == PCE_STATUS_ENABLED) { + if (ce->enabled_when_prepared) { + clk_disable_unprepare(ce->clk); + ce->status = PCE_STATUS_ACQUIRED; + } else { clk_disable(ce->clk); - ce->status = PCE_STATUS_ACQUIRED; + ce->status = PCE_STATUS_PREPARED; + } } } - spin_unlock_irqrestore(&psd->lock, flags); + pm_clk_op_unlock(psd, &flags); return 0; } @@ -428,18 +566,21 @@ int pm_clk_resume(struct device *dev) struct pm_subsys_data *psd = dev_to_psd(dev); struct pm_clock_entry *ce; unsigned long flags; + int ret; dev_dbg(dev, "%s()\n", __func__); if (!psd) return 0; - spin_lock_irqsave(&psd->lock, flags); + ret = pm_clk_op_lock(psd, &flags, __func__); + if (ret) + return ret; list_for_each_entry(ce, &psd->clock_list, node) __pm_clk_enable(dev, ce); - spin_unlock_irqrestore(&psd->lock, flags); + pm_clk_op_unlock(psd, &flags); return 0; } diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 8c1d04db990d..3d751ae5bc70 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -1164,6 +1164,27 @@ int clk_enable(struct clk *clk) } EXPORT_SYMBOL_GPL(clk_enable); +/** + * clk_is_enabled_when_prepared - indicate if preparing a clock also enables it. + * @clk: clock source + * + * Returns true if clk_prepare() implicitly enables the clock, effectively + * making clk_enable()/clk_disable() no-ops, false otherwise. + * + * This is of interest mainly to power management code where actually + * disabling the clock also requires unpreparing it to have any material + * effect. + * + * Regardless of the value returned here, the caller must always invoke + * clk_enable() or clk_prepare_enable() and counterparts for usage counts + * to be right. + */ +bool clk_is_enabled_when_prepared(struct clk *clk) +{ + return clk && !(clk->core->ops->enable && clk->core->ops->disable); +} +EXPORT_SYMBOL_GPL(clk_is_enabled_when_prepared); + static int clk_core_prepare_enable(struct clk_core *core) { int ret; diff --git a/include/linux/clk.h b/include/linux/clk.h index 31ff1bf1b79f..a4a86aa8b11a 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -238,6 +238,7 @@ static inline bool clk_is_match(const struct clk *p, const struct clk *q) #endif +#ifdef CONFIG_HAVE_CLK_PREPARE /** * clk_prepare - prepare a clock source * @clk: clock source @@ -246,10 +247,26 @@ static inline bool clk_is_match(const struct clk *p, const struct clk *q) * * Must not be called from within atomic context. */ -#ifdef CONFIG_HAVE_CLK_PREPARE int clk_prepare(struct clk *clk); int __must_check clk_bulk_prepare(int num_clks, const struct clk_bulk_data *clks); + +/** + * clk_is_enabled_when_prepared - indicate if preparing a clock also enables it. + * @clk: clock source + * + * Returns true if clk_prepare() implicitly enables the clock, effectively + * making clk_enable()/clk_disable() no-ops, false otherwise. + * + * This is of interest mainly to the power management code where actually + * disabling the clock also requires unpreparing it to have any material + * effect. + * + * Regardless of the value returned here, the caller must always invoke + * clk_enable() or clk_prepare_enable() and counterparts for usage counts + * to be right. + */ +bool clk_is_enabled_when_prepared(struct clk *clk); #else static inline int clk_prepare(struct clk *clk) { @@ -263,6 +280,11 @@ clk_bulk_prepare(int num_clks, const struct clk_bulk_data *clks) might_sleep(); return 0; } + +static inline bool clk_is_enabled_when_prepared(struct clk *clk) +{ + return false; +} #endif /** diff --git a/include/linux/pm.h b/include/linux/pm.h index 47aca6bac1d6..482313a8ccfc 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -537,6 +537,8 @@ struct pm_subsys_data { spinlock_t lock; unsigned int refcount; #ifdef CONFIG_PM_CLK + unsigned int clock_op_might_sleep; + struct mutex clock_mutex; struct list_head clock_list; #endif #ifdef CONFIG_PM_GENERIC_DOMAINS -- cgit v1.2.3 From 3fde13f817e23f05ce407d136325df4cbc913e67 Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Fri, 22 Jan 2021 17:46:43 +0800 Subject: f2fs: compress: support compress level Expand 'compress_algorithm' mount option to accept parameter as format of :, by this way, it gives a way to allow user to do more specified config on lz4 and zstd compression level, then f2fs compression can provide higher compress ratio. In order to set compress level for lz4 algorithm, it needs to set CONFIG_LZ4HC_COMPRESS and CONFIG_F2FS_FS_LZ4HC config to enable lz4hc compress algorithm. CR and performance number on lz4/lz4hc algorithm: dd if=enwik9 of=compressed_file conv=fsync Original blocks: 244382 lz4 lz4hc-9 compressed blocks 170647 163270 compress ratio 69.8% 66.8% speed 16.4207 s, 60.9 MB/s 26.7299 s, 37.4 MB/s compress ratio = after / before Signed-off-by: Chao Yu Signed-off-by: Jaegeuk Kim --- Documentation/filesystems/f2fs.rst | 5 +++ fs/f2fs/Kconfig | 10 +++++ fs/f2fs/compress.c | 41 ++++++++++++++++-- fs/f2fs/f2fs.h | 9 ++++ fs/f2fs/super.c | 89 +++++++++++++++++++++++++++++++++++++- include/linux/f2fs_fs.h | 3 ++ 6 files changed, 152 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index dae15c96e659..5eff4009e77e 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -249,6 +249,11 @@ checkpoint=%s[:%u[%]] Set to "disable" to turn off checkpointing. Set to "enabl This space is reclaimed once checkpoint=enable. compress_algorithm=%s Control compress algorithm, currently f2fs supports "lzo", "lz4", "zstd" and "lzo-rle" algorithm. +compress_algorithm=%s:%d Control compress algorithm and its compress level, now, only + "lz4" and "zstd" support compress level config. + algorithm level range + lz4 3 - 16 + zstd 1 - 22 compress_log_size=%u Support configuring compress cluster size, the size will be 4KB * (1 << %u), 16KB is minimum size, also it's default size. diff --git a/fs/f2fs/Kconfig b/fs/f2fs/Kconfig index d13c5c6a9787..63c1fc1a0e3b 100644 --- a/fs/f2fs/Kconfig +++ b/fs/f2fs/Kconfig @@ -119,6 +119,16 @@ config F2FS_FS_LZ4 help Support LZ4 compress algorithm, if unsure, say Y. +config F2FS_FS_LZ4HC + bool "LZ4HC compression support" + depends on F2FS_FS_COMPRESSION + depends on F2FS_FS_LZ4 + select LZ4HC_COMPRESS + default y + help + Support LZ4HC compress algorithm, LZ4HC has compatible on-disk + layout with LZ4, if unsure, say Y. + config F2FS_FS_ZSTD bool "ZSTD compression support" depends on F2FS_FS_COMPRESSION diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 4bcbacfe3325..a345a41e2119 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -252,8 +252,14 @@ static const struct f2fs_compress_ops f2fs_lzo_ops = { #ifdef CONFIG_F2FS_FS_LZ4 static int lz4_init_compress_ctx(struct compress_ctx *cc) { - cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), - LZ4_MEM_COMPRESS, GFP_NOFS); + unsigned int size = LZ4_MEM_COMPRESS; + +#ifdef CONFIG_F2FS_FS_LZ4HC + if (F2FS_I(cc->inode)->i_compress_flag >> COMPRESS_LEVEL_OFFSET) + size = LZ4HC_MEM_COMPRESS; +#endif + + cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), size, GFP_NOFS); if (!cc->private) return -ENOMEM; @@ -272,10 +278,34 @@ static void lz4_destroy_compress_ctx(struct compress_ctx *cc) cc->private = NULL; } +#ifdef CONFIG_F2FS_FS_LZ4HC +static int lz4hc_compress_pages(struct compress_ctx *cc) +{ + unsigned char level = F2FS_I(cc->inode)->i_compress_flag >> + COMPRESS_LEVEL_OFFSET; + int len; + + if (level) + len = LZ4_compress_HC(cc->rbuf, cc->cbuf->cdata, cc->rlen, + cc->clen, level, cc->private); + else + len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen, + cc->clen, cc->private); + if (!len) + return -EAGAIN; + + cc->clen = len; + return 0; +} +#endif + static int lz4_compress_pages(struct compress_ctx *cc) { int len; +#ifdef CONFIG_F2FS_FS_LZ4HC + return lz4hc_compress_pages(cc); +#endif len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen, cc->clen, cc->private); if (!len) @@ -325,8 +355,13 @@ static int zstd_init_compress_ctx(struct compress_ctx *cc) ZSTD_CStream *stream; void *workspace; unsigned int workspace_size; + unsigned char level = F2FS_I(cc->inode)->i_compress_flag >> + COMPRESS_LEVEL_OFFSET; + + if (!level) + level = F2FS_ZSTD_DEFAULT_CLEVEL; - params = ZSTD_getParams(F2FS_ZSTD_DEFAULT_CLEVEL, cc->rlen, 0); + params = ZSTD_getParams(level, cc->rlen, 0); workspace_size = ZSTD_CStreamWorkspaceBound(params.cParams); workspace = f2fs_kvmalloc(F2FS_I_SB(cc->inode), diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index bb11759191dc..36012181c17f 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -146,6 +146,7 @@ struct f2fs_mount_info { /* For compression */ unsigned char compress_algorithm; /* algorithm type */ unsigned char compress_log_size; /* cluster log size */ + unsigned char compress_level; /* compress level */ bool compress_chksum; /* compressed data chksum */ unsigned char compress_ext_cnt; /* extension count */ int compress_mode; /* compression mode */ @@ -735,6 +736,7 @@ struct f2fs_inode_info { atomic_t i_compr_blocks; /* # of compressed blocks */ unsigned char i_compress_algorithm; /* algorithm type */ unsigned char i_log_cluster_size; /* log of cluster size */ + unsigned char i_compress_level; /* compress level (lz4hc,zstd) */ unsigned short i_compress_flag; /* compress flag */ unsigned int i_cluster_size; /* cluster size */ }; @@ -1310,6 +1312,8 @@ struct compress_data { #define F2FS_COMPRESSED_PAGE_MAGIC 0xF5F2C000 +#define COMPRESS_LEVEL_OFFSET 8 + /* compress context */ struct compress_ctx { struct inode *inode; /* inode the context belong to */ @@ -3934,6 +3938,11 @@ static inline void set_compress_context(struct inode *inode) 1 << COMPRESS_CHKSUM : 0; F2FS_I(inode)->i_cluster_size = 1 << F2FS_I(inode)->i_log_cluster_size; + if (F2FS_I(inode)->i_compress_algorithm == COMPRESS_LZ4 && + F2FS_OPTION(sbi).compress_level) + F2FS_I(inode)->i_compress_flag |= + F2FS_OPTION(sbi).compress_level << + COMPRESS_LEVEL_OFFSET; F2FS_I(inode)->i_flags |= F2FS_COMPR_FL; set_inode_flag(inode, FI_COMPRESSED_FILE); stat_inc_compr_inode(inode); diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index a275bd312ae5..c8be27a9eed6 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include "f2fs.h" #include "node.h" @@ -464,6 +466,74 @@ static int f2fs_set_test_dummy_encryption(struct super_block *sb, return 0; } +#ifdef CONFIG_F2FS_FS_COMPRESSION +#ifdef CONFIG_F2FS_FS_LZ4 +static int f2fs_set_lz4hc_level(struct f2fs_sb_info *sbi, const char *str) +{ +#ifdef CONFIG_F2FS_FS_LZ4HC + unsigned int level; +#endif + + if (strlen(str) == 3) { + F2FS_OPTION(sbi).compress_level = 0; + return 0; + } + +#ifdef CONFIG_F2FS_FS_LZ4HC + str += 3; + + if (str[0] != ':') { + f2fs_info(sbi, "wrong format, e.g. :"); + return -EINVAL; + } + if (kstrtouint(str + 1, 10, &level)) + return -EINVAL; + + if (level < LZ4HC_MIN_CLEVEL || level > LZ4HC_MAX_CLEVEL) { + f2fs_info(sbi, "invalid lz4hc compress level: %d", level); + return -EINVAL; + } + + F2FS_OPTION(sbi).compress_level = level; + return 0; +#else + f2fs_info(sbi, "kernel doesn't support lz4hc compression"); + return -EINVAL; +#endif +} +#endif + +#ifdef CONFIG_F2FS_FS_ZSTD +static int f2fs_set_zstd_level(struct f2fs_sb_info *sbi, const char *str) +{ + unsigned int level; + int len = 4; + + if (strlen(str) == len) { + F2FS_OPTION(sbi).compress_level = 0; + return 0; + } + + str += len; + + if (str[0] != ':') { + f2fs_info(sbi, "wrong format, e.g. :"); + return -EINVAL; + } + if (kstrtouint(str + 1, 10, &level)) + return -EINVAL; + + if (!level || level > ZSTD_maxCLevel()) { + f2fs_info(sbi, "invalid zstd compress level: %d", level); + return -EINVAL; + } + + F2FS_OPTION(sbi).compress_level = level; + return 0; +} +#endif +#endif + static int parse_options(struct super_block *sb, char *options, bool is_remount) { struct f2fs_sb_info *sbi = F2FS_SB(sb); @@ -883,20 +953,31 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) return -ENOMEM; if (!strcmp(name, "lzo")) { #ifdef CONFIG_F2FS_FS_LZO + F2FS_OPTION(sbi).compress_level = 0; F2FS_OPTION(sbi).compress_algorithm = COMPRESS_LZO; #else f2fs_info(sbi, "kernel doesn't support lzo compression"); #endif - } else if (!strcmp(name, "lz4")) { + } else if (!strncmp(name, "lz4", 3)) { #ifdef CONFIG_F2FS_FS_LZ4 + ret = f2fs_set_lz4hc_level(sbi, name); + if (ret) { + kfree(name); + return -EINVAL; + } F2FS_OPTION(sbi).compress_algorithm = COMPRESS_LZ4; #else f2fs_info(sbi, "kernel doesn't support lz4 compression"); #endif - } else if (!strcmp(name, "zstd")) { + } else if (!strncmp(name, "zstd", 4)) { #ifdef CONFIG_F2FS_FS_ZSTD + ret = f2fs_set_zstd_level(sbi, name); + if (ret) { + kfree(name); + return -EINVAL; + } F2FS_OPTION(sbi).compress_algorithm = COMPRESS_ZSTD; #else @@ -904,6 +985,7 @@ static int parse_options(struct super_block *sb, char *options, bool is_remount) #endif } else if (!strcmp(name, "lzo-rle")) { #ifdef CONFIG_F2FS_FS_LZORLE + F2FS_OPTION(sbi).compress_level = 0; F2FS_OPTION(sbi).compress_algorithm = COMPRESS_LZORLE; #else @@ -1555,6 +1637,9 @@ static inline void f2fs_show_compress_options(struct seq_file *seq, } seq_printf(seq, ",compress_algorithm=%s", algtype); + if (F2FS_OPTION(sbi).compress_level) + seq_printf(seq, ":%d", F2FS_OPTION(sbi).compress_level); + seq_printf(seq, ",compress_log_size=%u", F2FS_OPTION(sbi).compress_log_size); diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index 7dc2a06cf19a..c6cc0a566ef5 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -274,6 +274,9 @@ struct f2fs_inode { __u8 i_compress_algorithm; /* compress algorithm */ __u8 i_log_cluster_size; /* log of cluster size */ __le16 i_compress_flag; /* compress flag */ + /* 0 bit: chksum flag + * [10,15] bits: compress level + */ __le32 i_extra_end[0]; /* for attribute size calculation */ } __packed; __le32 i_addr[DEF_ADDRS_PER_INODE]; /* Pointers to data blocks */ -- cgit v1.2.3 From 794c43f716845e2d48ce195ed5c4179a4e05ce5f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Dec 2020 15:25:29 -0800 Subject: libfs: unexport generic_ci_d_compare() and generic_ci_d_hash() Now that generic_set_encrypted_ci_d_ops() has been added and ext4 and f2fs are using it, it's no longer necessary to export generic_ci_d_compare() and generic_ci_d_hash() to filesystems. Reviewed-by: Gabriel Krisman Bertazi Signed-off-by: Eric Biggers Signed-off-by: Jaegeuk Kim --- fs/libfs.c | 8 +++----- include/linux/fs.h | 5 ----- 2 files changed, 3 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/fs/libfs.c b/fs/libfs.c index d1c3bade9f30..79721571e014 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -1388,8 +1388,8 @@ static bool needs_casefold(const struct inode *dir) * * Return: 0 if names match, 1 if mismatch, or -ERRNO */ -int generic_ci_d_compare(const struct dentry *dentry, unsigned int len, - const char *str, const struct qstr *name) +static int generic_ci_d_compare(const struct dentry *dentry, unsigned int len, + const char *str, const struct qstr *name) { const struct dentry *parent = READ_ONCE(dentry->d_parent); const struct inode *dir = READ_ONCE(parent->d_inode); @@ -1426,7 +1426,6 @@ fallback: return 1; return !!memcmp(str, name->name, len); } -EXPORT_SYMBOL(generic_ci_d_compare); /** * generic_ci_d_hash - generic d_hash implementation for casefolding filesystems @@ -1435,7 +1434,7 @@ EXPORT_SYMBOL(generic_ci_d_compare); * * Return: 0 if hash was successful or unchanged, and -EINVAL on error */ -int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str) +static int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str) { const struct inode *dir = READ_ONCE(dentry->d_inode); struct super_block *sb = dentry->d_sb; @@ -1450,7 +1449,6 @@ int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str) return -EINVAL; return 0; } -EXPORT_SYMBOL(generic_ci_d_hash); static const struct dentry_operations generic_ci_dentry_ops = { .d_hash = generic_ci_d_hash, diff --git a/include/linux/fs.h b/include/linux/fs.h index fd47deea7c17..6d8b1e7337e4 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3192,11 +3192,6 @@ extern int generic_file_fsync(struct file *, loff_t, loff_t, int); extern int generic_check_addressable(unsigned, u64); -#ifdef CONFIG_UNICODE -extern int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str); -extern int generic_ci_d_compare(const struct dentry *dentry, unsigned int len, - const char *str, const struct qstr *name); -#endif extern void generic_set_encrypted_ci_d_ops(struct dentry *dentry); #ifdef CONFIG_MIGRATION -- cgit v1.2.3 From 8063e184e49011f6f3f34f6c358dc8a83890bb5b Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Wed, 27 Jan 2021 14:15:01 -0800 Subject: skmsg: Make sk_psock_destroy() static sk_psock_destroy() is a RCU callback, I can't see any reason why it could be used outside. Signed-off-by: Cong Wang Signed-off-by: Daniel Borkmann Cc: John Fastabend Cc: Jakub Sitnicki Cc: Lorenz Bauer Link: https://lore.kernel.org/bpf/20210127221501.46866-1-xiyou.wangcong@gmail.com --- include/linux/skmsg.h | 1 - net/core/skmsg.c | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index fec0c5ac1c4f..8edbbf5f2f93 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -390,7 +390,6 @@ static inline struct sk_psock *sk_psock_get(struct sock *sk) } void sk_psock_stop(struct sock *sk, struct sk_psock *psock); -void sk_psock_destroy(struct rcu_head *rcu); void sk_psock_drop(struct sock *sk, struct sk_psock *psock); static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock) diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 25cdbb20f3a0..1261512d6807 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -669,14 +669,13 @@ static void sk_psock_destroy_deferred(struct work_struct *gc) kfree(psock); } -void sk_psock_destroy(struct rcu_head *rcu) +static void sk_psock_destroy(struct rcu_head *rcu) { struct sk_psock *psock = container_of(rcu, struct sk_psock, rcu); INIT_WORK(&psock->gc, sk_psock_destroy_deferred); schedule_work(&psock->gc); } -EXPORT_SYMBOL_GPL(sk_psock_destroy); void sk_psock_drop(struct sock *sk, struct sk_psock *psock) { -- cgit v1.2.3 From 772412176fb98493158929b220fe250127f611af Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Wed, 27 Jan 2021 11:31:39 -0800 Subject: bpf: Allow rewriting to ports under ip_unprivileged_port_start At the moment, BPF_CGROUP_INET{4,6}_BIND hooks can rewrite user_port to the privileged ones (< ip_unprivileged_port_start), but it will be rejected later on in the __inet_bind or __inet6_bind. Let's add another return value to indicate that CAP_NET_BIND_SERVICE check should be ignored. Use the same idea as we currently use in cgroup/egress where bit #1 indicates CN. Instead, for cgroup/bind{4,6}, bit #1 indicates that CAP_NET_BIND_SERVICE should be bypassed. v5: - rename flags to be less confusing (Andrey Ignatov) - rework BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY to work on flags and accept BPF_RET_SET_CN (no behavioral changes) v4: - Add missing IPv6 support (Martin KaFai Lau) v3: - Update description (Martin KaFai Lau) - Fix capability restore in selftest (Martin KaFai Lau) v2: - Switch to explicit return code (Martin KaFai Lau) Signed-off-by: Stanislav Fomichev Signed-off-by: Alexei Starovoitov Reviewed-by: Martin KaFai Lau Acked-by: Andrey Ignatov Link: https://lore.kernel.org/bpf/20210127193140.3170382-1-sdf@google.com --- include/linux/bpf-cgroup.h | 38 ++++++++++++++++++++++++--------- include/linux/bpf.h | 52 +++++++++++++++++++++++++++++----------------- include/net/inet_common.h | 2 ++ kernel/bpf/cgroup.c | 8 +++++-- kernel/bpf/verifier.c | 3 +++ net/ipv4/af_inet.c | 9 +++++--- net/ipv6/af_inet6.c | 9 +++++--- 7 files changed, 84 insertions(+), 37 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 0748fd87969e..c42e02b4d84b 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -125,7 +125,8 @@ int __cgroup_bpf_run_filter_sk(struct sock *sk, int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, struct sockaddr *uaddr, enum bpf_attach_type type, - void *t_ctx); + void *t_ctx, + u32 *flags); int __cgroup_bpf_run_filter_sock_ops(struct sock *sk, struct bpf_sock_ops_kern *sock_ops, @@ -231,30 +232,48 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type) \ ({ \ + u32 __unused_flags; \ int __ret = 0; \ if (cgroup_bpf_enabled(type)) \ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ - NULL); \ + NULL, \ + &__unused_flags); \ __ret; \ }) #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) \ ({ \ + u32 __unused_flags; \ int __ret = 0; \ if (cgroup_bpf_enabled(type)) { \ lock_sock(sk); \ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ - t_ctx); \ + t_ctx, \ + &__unused_flags); \ release_sock(sk); \ } \ __ret; \ }) -#define BPF_CGROUP_RUN_PROG_INET4_BIND_LOCK(sk, uaddr) \ - BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET4_BIND, NULL) - -#define BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK(sk, uaddr) \ - BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_BIND, NULL) +/* BPF_CGROUP_INET4_BIND and BPF_CGROUP_INET6_BIND can return extra flags + * via upper bits of return code. The only flag that is supported + * (at bit position 0) is to indicate CAP_NET_BIND_SERVICE capability check + * should be bypassed (BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE). + */ +#define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, type, bind_flags) \ +({ \ + u32 __flags = 0; \ + int __ret = 0; \ + if (cgroup_bpf_enabled(type)) { \ + lock_sock(sk); \ + __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ + NULL, &__flags); \ + release_sock(sk); \ + if (__flags & BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE) \ + *bind_flags |= BIND_NO_CAP_NET_BIND_SERVICE; \ + } \ + __ret; \ +}) #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) \ ((cgroup_bpf_enabled(BPF_CGROUP_INET4_CONNECT) || \ @@ -453,8 +472,7 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; }) #define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; }) #define BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk) ({ 0; }) -#define BPF_CGROUP_RUN_PROG_INET4_BIND_LOCK(sk, uaddr) ({ 0; }) -#define BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK(sk, uaddr) ({ 0; }) +#define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, type, flags) ({ 0; }) #define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; }) #define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; }) #define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; }) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 1aac2af12fed..321966fc35db 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1073,6 +1073,34 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, struct bpf_prog *include_prog, struct bpf_prog_array **new_array); +/* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */ +#define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0) +/* BPF program asks to set CN on the packet. */ +#define BPF_RET_SET_CN (1 << 0) + +#define BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, ret_flags) \ + ({ \ + struct bpf_prog_array_item *_item; \ + struct bpf_prog *_prog; \ + struct bpf_prog_array *_array; \ + u32 _ret = 1; \ + u32 func_ret; \ + migrate_disable(); \ + rcu_read_lock(); \ + _array = rcu_dereference(array); \ + _item = &_array->items[0]; \ + while ((_prog = READ_ONCE(_item->prog))) { \ + bpf_cgroup_storage_set(_item->cgroup_storage); \ + func_ret = func(_prog, ctx); \ + _ret &= (func_ret & 1); \ + *(ret_flags) |= (func_ret >> 1); \ + _item++; \ + } \ + rcu_read_unlock(); \ + migrate_enable(); \ + _ret; \ + }) + #define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \ ({ \ struct bpf_prog_array_item *_item; \ @@ -1120,25 +1148,11 @@ _out: \ */ #define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \ ({ \ - struct bpf_prog_array_item *_item; \ - struct bpf_prog *_prog; \ - struct bpf_prog_array *_array; \ - u32 ret; \ - u32 _ret = 1; \ - u32 _cn = 0; \ - migrate_disable(); \ - rcu_read_lock(); \ - _array = rcu_dereference(array); \ - _item = &_array->items[0]; \ - while ((_prog = READ_ONCE(_item->prog))) { \ - bpf_cgroup_storage_set(_item->cgroup_storage); \ - ret = func(_prog, ctx); \ - _ret &= (ret & 1); \ - _cn |= (ret & 2); \ - _item++; \ - } \ - rcu_read_unlock(); \ - migrate_enable(); \ + u32 _flags = 0; \ + bool _cn; \ + u32 _ret; \ + _ret = BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, &_flags); \ + _cn = _flags & BPF_RET_SET_CN; \ if (_ret) \ _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \ else \ diff --git a/include/net/inet_common.h b/include/net/inet_common.h index cb2818862919..cad2a611efde 100644 --- a/include/net/inet_common.h +++ b/include/net/inet_common.h @@ -41,6 +41,8 @@ int inet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len); #define BIND_WITH_LOCK (1 << 1) /* Called from BPF program. */ #define BIND_FROM_BPF (1 << 2) +/* Skip CAP_NET_BIND_SERVICE check. */ +#define BIND_NO_CAP_NET_BIND_SERVICE (1 << 3) int __inet_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len, u32 flags); int inet_getname(struct socket *sock, struct sockaddr *uaddr, diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index da649f20d6b2..cdf3c7e611d9 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1055,6 +1055,8 @@ EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk); * @uaddr: sockaddr struct provided by user * @type: The type of program to be exectuted * @t_ctx: Pointer to attach type specific context + * @flags: Pointer to u32 which contains higher bits of BPF program + * return value (OR'ed together). * * socket is expected to be of type INET or INET6. * @@ -1064,7 +1066,8 @@ EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk); int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, struct sockaddr *uaddr, enum bpf_attach_type type, - void *t_ctx) + void *t_ctx, + u32 *flags) { struct bpf_sock_addr_kern ctx = { .sk = sk, @@ -1087,7 +1090,8 @@ int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, } cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); - ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN); + ret = BPF_PROG_RUN_ARRAY_FLAGS(cgrp->bpf.effective[type], &ctx, + BPF_PROG_RUN, flags); return ret == 1 ? 0 : -EPERM; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d0eae51b31e4..972fc38eb62d 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -7986,6 +7986,9 @@ static int check_return_code(struct bpf_verifier_env *env) env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) range = tnum_range(1, 1); + if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || + env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) + range = tnum_range(0, 3); break; case BPF_PROG_TYPE_CGROUP_SKB: if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 6ba2930ff49b..aaa94bea19c3 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -438,6 +438,7 @@ EXPORT_SYMBOL(inet_release); int inet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) { struct sock *sk = sock->sk; + u32 flags = BIND_WITH_LOCK; int err; /* If the socket has its own bind function then use it. (RAW) */ @@ -450,11 +451,12 @@ int inet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) /* BPF prog is run before any checks are done so that if the prog * changes context in a wrong way it will be caught. */ - err = BPF_CGROUP_RUN_PROG_INET4_BIND_LOCK(sk, uaddr); + err = BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, + BPF_CGROUP_INET4_BIND, &flags); if (err) return err; - return __inet_bind(sk, uaddr, addr_len, BIND_WITH_LOCK); + return __inet_bind(sk, uaddr, addr_len, flags); } EXPORT_SYMBOL(inet_bind); @@ -499,7 +501,8 @@ int __inet_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len, snum = ntohs(addr->sin_port); err = -EACCES; - if (snum && inet_port_requires_bind_service(net, snum) && + if (!(flags & BIND_NO_CAP_NET_BIND_SERVICE) && + snum && inet_port_requires_bind_service(net, snum) && !ns_capable(net->user_ns, CAP_NET_BIND_SERVICE)) goto out; diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index b9c654836b72..f091fe9b4da5 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -295,7 +295,8 @@ static int __inet6_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len, return -EINVAL; snum = ntohs(addr->sin6_port); - if (snum && inet_port_requires_bind_service(net, snum) && + if (!(flags & BIND_NO_CAP_NET_BIND_SERVICE) && + snum && inet_port_requires_bind_service(net, snum) && !ns_capable(net->user_ns, CAP_NET_BIND_SERVICE)) return -EACCES; @@ -439,6 +440,7 @@ out_unlock: int inet6_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) { struct sock *sk = sock->sk; + u32 flags = BIND_WITH_LOCK; int err = 0; /* If the socket has its own bind function then use it. */ @@ -451,11 +453,12 @@ int inet6_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) /* BPF prog is run before any checks are done so that if the prog * changes context in a wrong way it will be caught. */ - err = BPF_CGROUP_RUN_PROG_INET6_BIND_LOCK(sk, uaddr); + err = BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, + BPF_CGROUP_INET6_BIND, &flags); if (err) return err; - return __inet6_bind(sk, uaddr, addr_len, BIND_WITH_LOCK); + return __inet6_bind(sk, uaddr, addr_len, flags); } EXPORT_SYMBOL(inet6_bind); -- cgit v1.2.3 From 3d347b1b19da20f973d1d3c6bb60c11185606afd Mon Sep 17 00:00:00 2001 From: Aya Levin Date: Tue, 26 Jan 2021 15:24:07 -0800 Subject: net/mlx5: Add support for devlink traps in mlx5 core driver Add devlink traps infra-structure to mlx5 core driver. Add traps list to mlx5_priv and corresponding API: - mlx5_devlink_trap_report() to wrap trap reports to devlink - mlx5_devlink_trap_get_num_active() to decide whether to open/close trap resources. Signed-off-by: Aya Levin Reviewed-by: Tariq Toukan Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/devlink.c | 84 +++++++++++++++++++++++ drivers/net/ethernet/mellanox/mlx5/core/devlink.h | 16 +++++ drivers/net/ethernet/mellanox/mlx5/core/main.c | 2 + include/linux/mlx5/driver.h | 1 + 4 files changed, 103 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index 3261d0dc1104..f04afaf785cb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -168,6 +168,56 @@ static int mlx5_devlink_reload_up(struct devlink *devlink, enum devlink_reload_a return 0; } +static struct mlx5_devlink_trap *mlx5_find_trap_by_id(struct mlx5_core_dev *dev, int trap_id) +{ + struct mlx5_devlink_trap *dl_trap; + + list_for_each_entry(dl_trap, &dev->priv.traps, list) + if (dl_trap->trap.id == trap_id) + return dl_trap; + + return NULL; +} + +static int mlx5_devlink_trap_init(struct devlink *devlink, const struct devlink_trap *trap, + void *trap_ctx) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + struct mlx5_devlink_trap *dl_trap; + + dl_trap = kzalloc(sizeof(*dl_trap), GFP_KERNEL); + if (!dl_trap) + return -ENOMEM; + + dl_trap->trap.id = trap->id; + dl_trap->trap.action = DEVLINK_TRAP_ACTION_DROP; + dl_trap->item = trap_ctx; + + if (mlx5_find_trap_by_id(dev, trap->id)) { + kfree(dl_trap); + mlx5_core_err(dev, "Devlink trap: Trap 0x%x already found", trap->id); + return -EEXIST; + } + + list_add_tail(&dl_trap->list, &dev->priv.traps); + return 0; +} + +static void mlx5_devlink_trap_fini(struct devlink *devlink, const struct devlink_trap *trap, + void *trap_ctx) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + struct mlx5_devlink_trap *dl_trap; + + dl_trap = mlx5_find_trap_by_id(dev, trap->id); + if (!dl_trap) { + mlx5_core_err(dev, "Devlink trap: Missing trap id 0x%x", trap->id); + return; + } + list_del(&dl_trap->list); + kfree(dl_trap); +} + static const struct devlink_ops mlx5_devlink_ops = { #ifdef CONFIG_MLX5_ESWITCH .eswitch_mode_set = mlx5_devlink_eswitch_mode_set, @@ -186,8 +236,42 @@ static const struct devlink_ops mlx5_devlink_ops = { .reload_limits = BIT(DEVLINK_RELOAD_LIMIT_NO_RESET), .reload_down = mlx5_devlink_reload_down, .reload_up = mlx5_devlink_reload_up, + .trap_init = mlx5_devlink_trap_init, + .trap_fini = mlx5_devlink_trap_fini, }; +void mlx5_devlink_trap_report(struct mlx5_core_dev *dev, int trap_id, struct sk_buff *skb, + struct devlink_port *dl_port) +{ + struct devlink *devlink = priv_to_devlink(dev); + struct mlx5_devlink_trap *dl_trap; + + dl_trap = mlx5_find_trap_by_id(dev, trap_id); + if (!dl_trap) { + mlx5_core_err(dev, "Devlink trap: Report on invalid trap id 0x%x", trap_id); + return; + } + + if (dl_trap->trap.action != DEVLINK_TRAP_ACTION_TRAP) { + mlx5_core_dbg(dev, "Devlink trap: Trap id %d has action %d", trap_id, + dl_trap->trap.action); + return; + } + devlink_trap_report(devlink, skb, dl_trap->item, dl_port, NULL); +} + +int mlx5_devlink_trap_get_num_active(struct mlx5_core_dev *dev) +{ + struct mlx5_devlink_trap *dl_trap; + int count = 0; + + list_for_each_entry(dl_trap, &dev->priv.traps, list) + if (dl_trap->trap.action == DEVLINK_TRAP_ACTION_TRAP) + count++; + + return count; +} + struct devlink *mlx5_devlink_alloc(void) { return devlink_alloc(&mlx5_devlink_ops, sizeof(struct mlx5_core_dev)); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h index f0de327a59be..a9829006fa78 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h @@ -12,6 +12,22 @@ enum mlx5_devlink_param_id { MLX5_DEVLINK_PARAM_ID_ESW_LARGE_GROUP_NUM, }; +struct mlx5_trap_ctx { + int id; + int action; +}; + +struct mlx5_devlink_trap { + struct mlx5_trap_ctx trap; + void *item; + struct list_head list; +}; + +struct mlx5_core_dev; +void mlx5_devlink_trap_report(struct mlx5_core_dev *dev, int trap_id, struct sk_buff *skb, + struct devlink_port *dl_port); +int mlx5_devlink_trap_get_num_active(struct mlx5_core_dev *dev); + struct devlink *mlx5_devlink_alloc(void); void mlx5_devlink_free(struct devlink *devlink); int mlx5_devlink_register(struct devlink *devlink, struct device *dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index ca6f2fc39ea0..bfedf064db1a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -1305,6 +1305,8 @@ static int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) priv->dbg_root = debugfs_create_dir(dev_name(dev->device), mlx5_debugfs_root); + INIT_LIST_HEAD(&priv->traps); + err = mlx5_health_init(dev); if (err) goto err_health_init; diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index f93bfe7473aa..c4615dc51b6f 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -564,6 +564,7 @@ struct mlx5_priv { int host_pf_pages; struct mlx5_core_health health; + struct list_head traps; /* start: qp staff */ struct dentry *qp_debugfs; -- cgit v1.2.3 From 241dc159391fb9d351362d911a39dff84074cc92 Mon Sep 17 00:00:00 2001 From: Aya Levin Date: Tue, 26 Jan 2021 15:24:11 -0800 Subject: net/mlx5: Notify on trap action by blocking event In order to allow mlx5 core driver to trigger synchronous operations to its consumers, add a blocking events handler. Add wrappers to blocking_notifier_[call_chain/chain_register/chain_unregister]. Add trap callback for action set and notify about this change. Following patches in the set add a listener for this event. Signed-off-by: Aya Levin Reviewed-by: Tariq Toukan Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/devlink.c | 36 +++++++++++++++++++++++ drivers/net/ethernet/mellanox/mlx5/core/events.c | 28 ++++++++++++++++++ include/linux/mlx5/device.h | 4 +++ include/linux/mlx5/driver.h | 15 ++++++++++ 4 files changed, 83 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index f081eff9be25..c47291467cb0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -218,6 +218,41 @@ static void mlx5_devlink_trap_fini(struct devlink *devlink, const struct devlink kfree(dl_trap); } +static int mlx5_devlink_trap_action_set(struct devlink *devlink, + const struct devlink_trap *trap, + enum devlink_trap_action action, + struct netlink_ext_ack *extack) +{ + struct mlx5_core_dev *dev = devlink_priv(devlink); + enum devlink_trap_action action_orig; + struct mlx5_devlink_trap *dl_trap; + int err = 0; + + dl_trap = mlx5_find_trap_by_id(dev, trap->id); + if (!dl_trap) { + mlx5_core_err(dev, "Devlink trap: Set action on invalid trap id 0x%x", trap->id); + err = -EINVAL; + goto out; + } + + if (action != DEVLINK_TRAP_ACTION_DROP && action != DEVLINK_TRAP_ACTION_TRAP) { + err = -EOPNOTSUPP; + goto out; + } + + if (action == dl_trap->trap.action) + goto out; + + action_orig = dl_trap->trap.action; + dl_trap->trap.action = action; + err = mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_TYPE_TRAP, + &dl_trap->trap); + if (err) + dl_trap->trap.action = action_orig; +out: + return err; +} + static const struct devlink_ops mlx5_devlink_ops = { #ifdef CONFIG_MLX5_ESWITCH .eswitch_mode_set = mlx5_devlink_eswitch_mode_set, @@ -238,6 +273,7 @@ static const struct devlink_ops mlx5_devlink_ops = { .reload_up = mlx5_devlink_reload_up, .trap_init = mlx5_devlink_trap_init, .trap_fini = mlx5_devlink_trap_fini, + .trap_action_set = mlx5_devlink_trap_action_set, }; void mlx5_devlink_trap_report(struct mlx5_core_dev *dev, int trap_id, struct sk_buff *skb, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/events.c b/drivers/net/ethernet/mellanox/mlx5/core/events.c index 054c0bc36d24..670f25f5ffd1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/events.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/events.c @@ -61,6 +61,8 @@ struct mlx5_events { struct mlx5_pme_stats pme_stats; /*pcie_core*/ struct work_struct pcie_core_work; + /* driver notifier chain for sw events */ + struct blocking_notifier_head sw_nh; }; static const char *eqe_type_str(u8 type) @@ -351,6 +353,7 @@ int mlx5_events_init(struct mlx5_core_dev *dev) return -ENOMEM; } INIT_WORK(&events->pcie_core_work, mlx5_pcie_event); + BLOCKING_INIT_NOTIFIER_HEAD(&events->sw_nh); return 0; } @@ -406,3 +409,28 @@ int mlx5_notifier_call_chain(struct mlx5_events *events, unsigned int event, voi { return atomic_notifier_call_chain(&events->fw_nh, event, data); } + +/* This API is used only for processing and forwarding driver-specific + * events to mlx5 consumers. + */ +int mlx5_blocking_notifier_register(struct mlx5_core_dev *dev, struct notifier_block *nb) +{ + struct mlx5_events *events = dev->priv.events; + + return blocking_notifier_chain_register(&events->sw_nh, nb); +} + +int mlx5_blocking_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *nb) +{ + struct mlx5_events *events = dev->priv.events; + + return blocking_notifier_chain_unregister(&events->sw_nh, nb); +} + +int mlx5_blocking_notifier_call_chain(struct mlx5_core_dev *dev, unsigned int event, + void *data) +{ + struct mlx5_events *events = dev->priv.events; + + return blocking_notifier_call_chain(&events->sw_nh, event, data); +} diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index f1de49d64a98..77ba54d38772 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -359,6 +359,10 @@ enum mlx5_event { MLX5_EVENT_TYPE_MAX = 0x100, }; +enum mlx5_driver_event { + MLX5_DRIVER_EVENT_TYPE_TRAP = 0, +}; + enum { MLX5_TRACER_SUBTYPE_OWNERSHIP_CHANGE = 0x0, MLX5_TRACER_SUBTYPE_TRACES_AVAILABLE = 0x1, diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index c4615dc51b6f..45df5b465ba8 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1073,11 +1073,26 @@ enum { MAX_MR_CACHE_ENTRIES }; +/* Async-atomic event notifier used by mlx5 core to forward FW + * evetns recived from event queue to mlx5 consumers. + * Optimise event queue dipatching. + */ int mlx5_notifier_register(struct mlx5_core_dev *dev, struct notifier_block *nb); int mlx5_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *nb); + +/* Async-atomic event notifier used for forwarding + * evetns from the event queue into the to mlx5 events dispatcher, + * eswitch, clock and others. + */ int mlx5_eq_notifier_register(struct mlx5_core_dev *dev, struct mlx5_nb *nb); int mlx5_eq_notifier_unregister(struct mlx5_core_dev *dev, struct mlx5_nb *nb); +/* Blocking event notifier used to forward SW events, used for slow path */ +int mlx5_blocking_notifier_register(struct mlx5_core_dev *dev, struct notifier_block *nb); +int mlx5_blocking_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *nb); +int mlx5_blocking_notifier_call_chain(struct mlx5_core_dev *dev, unsigned int event, + void *data); + int mlx5_core_query_vendor_id(struct mlx5_core_dev *mdev, u32 *vendor_id); int mlx5_cmd_create_vport_lag(struct mlx5_core_dev *dev); -- cgit v1.2.3 From 5543e989fe5e2fe6a5829ee42c00152cac2bb8a0 Mon Sep 17 00:00:00 2001 From: Aya Levin Date: Tue, 26 Jan 2021 15:24:16 -0800 Subject: net/mlx5e: Add trap entity to ETH driver Introduce mlx5e_trap which includes a dedicated RQ and NAPI for trapped packets. Trap-RQ processes packets that were destined to be dropped, but for debug and visibility sake these packets are trapped and reported to devlink. Trap-RQ connects between the HW and the driver and is not a part of a channel. Open mlx5e_create_rq() and mlx5_core_destroy_rq() as API and add dedicate RQ handlers which report to devlink of trapped packets. Signed-off-by: Aya Levin Reviewed-by: Tariq Toukan Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/Makefile | 2 +- drivers/net/ethernet/mellanox/mlx5/core/en.h | 7 + drivers/net/ethernet/mellanox/mlx5/core/en/trap.c | 409 ++++++++++++++++++++++ drivers/net/ethernet/mellanox/mlx5/core/en/trap.h | 35 ++ drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 5 +- drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 46 +++ include/linux/mlx5/device.h | 5 + 7 files changed, 505 insertions(+), 4 deletions(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/trap.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/trap.h (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index fcfc0b114985..d44f5f6ee449 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -27,7 +27,7 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \ en_selftest.o en/port.o en/monitor_stats.o en/health.o \ en/reporter_tx.o en/reporter_rx.o en/params.o en/xsk/pool.o \ en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o en/ptp.o \ - en/qos.o + en/qos.o en/trap.o # # Netdev extra diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index dc4895a1fa9b..f439a977ad61 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -564,6 +564,7 @@ typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq); typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16); int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk); +void mlx5e_rq_set_trap_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params); enum mlx5e_rq_flag { MLX5E_RQ_FLAG_XDP_XMIT, @@ -805,6 +806,8 @@ struct mlx5e_htb { u16 defcls; }; +struct mlx5e_trap; + struct mlx5e_priv { /* priv data path fields - start */ /* +1 for port ptp ts */ @@ -844,8 +847,10 @@ struct mlx5e_priv { struct mlx5_core_dev *mdev; struct net_device *netdev; + struct mlx5e_trap *en_trap; struct mlx5e_stats stats; struct mlx5e_channel_stats channel_stats[MLX5E_MAX_NUM_CHANNELS]; + struct mlx5e_channel_stats trap_stats; struct mlx5e_port_ptp_stats port_ptp_stats; u16 max_nch; u8 max_opened_tc; @@ -961,6 +966,8 @@ int mlx5e_open_rq(struct mlx5e_channel *c, struct mlx5e_params *params, int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time); void mlx5e_deactivate_rq(struct mlx5e_rq *rq); void mlx5e_close_rq(struct mlx5e_rq *rq); +int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param); +void mlx5e_destroy_rq(struct mlx5e_rq *rq); struct mlx5e_sq_param; int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c new file mode 100644 index 000000000000..5507efacb9dc --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2020 Mellanox Technologies */ + +#include +#include "en/txrx.h" +#include "en/params.h" +#include "en/trap.h" + +static int mlx5e_trap_napi_poll(struct napi_struct *napi, int budget) +{ + struct mlx5e_trap *trap_ctx = container_of(napi, struct mlx5e_trap, napi); + struct mlx5e_ch_stats *ch_stats = trap_ctx->stats; + struct mlx5e_rq *rq = &trap_ctx->rq; + bool busy = false; + int work_done = 0; + + ch_stats->poll++; + + work_done = mlx5e_poll_rx_cq(&rq->cq, budget); + busy |= work_done == budget; + busy |= rq->post_wqes(rq); + + if (busy) + return budget; + + if (unlikely(!napi_complete_done(napi, work_done))) + return work_done; + + mlx5e_cq_arm(&rq->cq); + return work_done; +} + +static int mlx5e_alloc_trap_rq(struct mlx5e_priv *priv, struct mlx5e_rq_param *rqp, + struct mlx5e_rq_stats *stats, struct mlx5e_params *params, + struct mlx5e_ch_stats *ch_stats, + struct mlx5e_rq *rq) +{ + void *rqc_wq = MLX5_ADDR_OF(rqc, rqp->rqc, wq); + struct mlx5_core_dev *mdev = priv->mdev; + struct page_pool_params pp_params = {}; + int node = dev_to_node(mdev->device); + u32 pool_size; + int wq_sz; + int err; + int i; + + rqp->wq.db_numa_node = node; + + rq->wq_type = params->rq_wq_type; + rq->pdev = mdev->device; + rq->netdev = priv->netdev; + rq->mdev = mdev; + rq->priv = priv; + rq->stats = stats; + rq->clock = &mdev->clock; + rq->tstamp = &priv->tstamp; + rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); + + xdp_rxq_info_unused(&rq->xdp_rxq); + + rq->buff.map_dir = DMA_FROM_DEVICE; + rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, NULL); + pool_size = 1 << params->log_rq_mtu_frames; + + err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq, &rq->wq_ctrl); + if (err) + return err; + + rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR]; + + wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); + + rq->wqe.info = rqp->frags_info; + rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride; + rq->wqe.frags = kvzalloc_node(array_size(sizeof(*rq->wqe.frags), + (wq_sz << rq->wqe.info.log_num_frags)), + GFP_KERNEL, node); + if (!rq->wqe.frags) { + err = -ENOMEM; + goto err_wq_cyc_destroy; + } + + err = mlx5e_init_di_list(rq, wq_sz, node); + if (err) + goto err_free_frags; + + rq->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key); + + mlx5e_rq_set_trap_handlers(rq, params); + + /* Create a page_pool and register it with rxq */ + pp_params.order = 0; + pp_params.flags = 0; /* No-internal DMA mapping in page_pool */ + pp_params.pool_size = pool_size; + pp_params.nid = node; + pp_params.dev = mdev->device; + pp_params.dma_dir = rq->buff.map_dir; + + /* page_pool can be used even when there is no rq->xdp_prog, + * given page_pool does not handle DMA mapping there is no + * required state to clear. And page_pool gracefully handle + * elevated refcnt. + */ + rq->page_pool = page_pool_create(&pp_params); + if (IS_ERR(rq->page_pool)) { + err = PTR_ERR(rq->page_pool); + rq->page_pool = NULL; + goto err_free_di_list; + } + for (i = 0; i < wq_sz; i++) { + struct mlx5e_rx_wqe_cyc *wqe = + mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i); + int f; + + for (f = 0; f < rq->wqe.info.num_frags; f++) { + u32 frag_size = rq->wqe.info.arr[f].frag_size | + MLX5_HW_START_PADDING; + + wqe->data[f].byte_count = cpu_to_be32(frag_size); + wqe->data[f].lkey = rq->mkey_be; + } + /* check if num_frags is not a pow of two */ + if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) { + wqe->data[f].byte_count = 0; + wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY); + wqe->data[f].addr = 0; + } + } + return 0; + +err_free_di_list: + mlx5e_free_di_list(rq); +err_free_frags: + kvfree(rq->wqe.frags); +err_wq_cyc_destroy: + mlx5_wq_destroy(&rq->wq_ctrl); + + return err; +} + +static void mlx5e_free_trap_rq(struct mlx5e_rq *rq) +{ + page_pool_destroy(rq->page_pool); + mlx5e_free_di_list(rq); + kvfree(rq->wqe.frags); + mlx5_wq_destroy(&rq->wq_ctrl); +} + +static int mlx5e_open_trap_rq(struct mlx5e_priv *priv, struct napi_struct *napi, + struct mlx5e_rq_stats *stats, struct mlx5e_params *params, + struct mlx5e_rq_param *rq_param, + struct mlx5e_ch_stats *ch_stats, + struct mlx5e_rq *rq) +{ + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_create_cq_param ccp = {}; + struct dim_cq_moder trap_moder = {}; + struct mlx5e_cq *cq = &rq->cq; + int err; + + ccp.node = dev_to_node(mdev->device); + ccp.ch_stats = ch_stats; + ccp.napi = napi; + ccp.ix = 0; + err = mlx5e_open_cq(priv, trap_moder, &rq_param->cqp, &ccp, cq); + if (err) + return err; + + err = mlx5e_alloc_trap_rq(priv, rq_param, stats, params, ch_stats, rq); + if (err) + goto err_destroy_cq; + + err = mlx5e_create_rq(rq, rq_param); + if (err) + goto err_free_rq; + + err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); + if (err) + goto err_destroy_rq; + + return 0; + +err_destroy_rq: + mlx5e_destroy_rq(rq); + mlx5e_free_rx_descs(rq); +err_free_rq: + mlx5e_free_trap_rq(rq); +err_destroy_cq: + mlx5e_close_cq(cq); + + return err; +} + +static void mlx5e_close_trap_rq(struct mlx5e_rq *rq) +{ + mlx5e_destroy_rq(rq); + mlx5e_free_rx_descs(rq); + mlx5e_free_trap_rq(rq); + mlx5e_close_cq(&rq->cq); +} + +static int mlx5e_create_trap_direct_rq_tir(struct mlx5_core_dev *mdev, struct mlx5e_tir *tir, + u32 rqn) +{ + void *tirc; + int inlen; + u32 *in; + int err; + + inlen = MLX5_ST_SZ_BYTES(create_tir_in); + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); + MLX5_SET(tirc, tirc, transport_domain, mdev->mlx5e_res.td.tdn); + MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_NONE); + MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT); + MLX5_SET(tirc, tirc, inline_rqn, rqn); + err = mlx5e_create_tir(mdev, tir, in); + kvfree(in); + + return err; +} + +static void mlx5e_destroy_trap_direct_rq_tir(struct mlx5_core_dev *mdev, struct mlx5e_tir *tir) +{ + mlx5e_destroy_tir(mdev, tir); +} + +static void mlx5e_activate_trap_rq(struct mlx5e_rq *rq) +{ + set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); +} + +static void mlx5e_deactivate_trap_rq(struct mlx5e_rq *rq) +{ + clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); +} + +static void mlx5e_build_trap_params(struct mlx5e_priv *priv, struct mlx5e_trap *t) +{ + struct mlx5e_params *params = &t->params; + + params->rq_wq_type = MLX5_WQ_TYPE_CYCLIC; + mlx5e_init_rq_type_params(priv->mdev, params); + params->sw_mtu = priv->netdev->max_mtu; + mlx5e_build_rq_param(priv, params, NULL, &t->rq_param); +} + +static struct mlx5e_trap *mlx5e_open_trap(struct mlx5e_priv *priv) +{ + int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(priv->mdev, 0)); + struct net_device *netdev = priv->netdev; + struct mlx5e_trap *t; + int err; + + t = kvzalloc_node(sizeof(*t), GFP_KERNEL, cpu_to_node(cpu)); + if (!t) + return ERR_PTR(-ENOMEM); + + mlx5e_build_trap_params(priv, t); + + t->priv = priv; + t->mdev = priv->mdev; + t->tstamp = &priv->tstamp; + t->pdev = mlx5_core_dma_dev(priv->mdev); + t->netdev = priv->netdev; + t->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key); + t->stats = &priv->trap_stats.ch; + + netif_napi_add(netdev, &t->napi, mlx5e_trap_napi_poll, 64); + + err = mlx5e_open_trap_rq(priv, &t->napi, + &priv->trap_stats.rq, + &t->params, &t->rq_param, + &priv->trap_stats.ch, + &t->rq); + if (unlikely(err)) + goto err_napi_del; + + err = mlx5e_create_trap_direct_rq_tir(t->mdev, &t->tir, t->rq.rqn); + if (err) + goto err_close_trap_rq; + + return t; + +err_close_trap_rq: + mlx5e_close_trap_rq(&t->rq); +err_napi_del: + netif_napi_del(&t->napi); + kvfree(t); + return ERR_PTR(err); +} + +void mlx5e_close_trap(struct mlx5e_trap *trap) +{ + mlx5e_destroy_trap_direct_rq_tir(trap->mdev, &trap->tir); + mlx5e_close_trap_rq(&trap->rq); + netif_napi_del(&trap->napi); + kvfree(trap); +} + +static void mlx5e_activate_trap(struct mlx5e_trap *trap) +{ + napi_enable(&trap->napi); + mlx5e_activate_trap_rq(&trap->rq); + napi_schedule(&trap->napi); +} + +void mlx5e_deactivate_trap(struct mlx5e_priv *priv) +{ + struct mlx5e_trap *trap = priv->en_trap; + + mlx5e_deactivate_trap_rq(&trap->rq); + napi_disable(&trap->napi); +} + +static struct mlx5e_trap *mlx5e_add_trap_queue(struct mlx5e_priv *priv) +{ + struct mlx5e_trap *trap; + + trap = mlx5e_open_trap(priv); + if (IS_ERR(trap)) + goto out; + + mlx5e_activate_trap(trap); +out: + return trap; +} + +static void mlx5e_del_trap_queue(struct mlx5e_priv *priv) +{ + mlx5e_deactivate_trap(priv); + mlx5e_close_trap(priv->en_trap); + priv->en_trap = NULL; +} + +static int mlx5e_trap_get_tirn(struct mlx5e_trap *en_trap) +{ + return en_trap->tir.tirn; +} + +static int mlx5e_handle_action_trap(struct mlx5e_priv *priv, int trap_id) +{ + bool open_queue = !priv->en_trap; + struct mlx5e_trap *trap; + int err; + + if (open_queue) { + trap = mlx5e_add_trap_queue(priv); + if (IS_ERR(trap)) + return PTR_ERR(trap); + priv->en_trap = trap; + } + + switch (trap_id) { + case DEVLINK_TRAP_GENERIC_ID_INGRESS_VLAN_FILTER: + err = mlx5e_add_vlan_trap(priv, trap_id, mlx5e_trap_get_tirn(priv->en_trap)); + if (err) + goto err_out; + break; + default: + netdev_warn(priv->netdev, "%s: Unknown trap id %d\n", __func__, trap_id); + err = -EINVAL; + goto err_out; + } + return 0; + +err_out: + if (open_queue) + mlx5e_del_trap_queue(priv); + return err; +} + +static int mlx5e_handle_action_drop(struct mlx5e_priv *priv, int trap_id) +{ + switch (trap_id) { + case DEVLINK_TRAP_GENERIC_ID_INGRESS_VLAN_FILTER: + mlx5e_remove_vlan_trap(priv); + break; + default: + netdev_warn(priv->netdev, "%s: Unknown trap id %d\n", __func__, trap_id); + return -EINVAL; + } + if (priv->en_trap && !mlx5_devlink_trap_get_num_active(priv->mdev)) + mlx5e_del_trap_queue(priv); + + return 0; +} + +int mlx5e_handle_trap_event(struct mlx5e_priv *priv, struct mlx5_trap_ctx *trap_ctx) +{ + int err = 0; + + switch (trap_ctx->action) { + case DEVLINK_TRAP_ACTION_TRAP: + err = mlx5e_handle_action_trap(priv, trap_ctx->id); + break; + case DEVLINK_TRAP_ACTION_DROP: + err = mlx5e_handle_action_drop(priv, trap_ctx->id); + break; + default: + netdev_warn(priv->netdev, "%s: Unsupported action %d\n", __func__, + trap_ctx->action); + err = -EINVAL; + } + return err; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.h b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.h new file mode 100644 index 000000000000..cc1fa9f12c45 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2020, Mellanox Technologies */ + +#ifndef __MLX5E_TRAP_H__ +#define __MLX5E_TRAP_H__ + +#include "../en.h" +#include "../devlink.h" + +struct mlx5e_trap { + /* data path */ + struct mlx5e_rq rq; + struct mlx5e_tir tir; + struct napi_struct napi; + struct device *pdev; + struct net_device *netdev; + __be32 mkey_be; + + /* data path - accessed per napi poll */ + struct mlx5e_ch_stats *stats; + + /* control */ + struct mlx5e_priv *priv; + struct mlx5_core_dev *mdev; + struct hwtstamp_config *tstamp; + DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES); + + struct mlx5e_params params; + struct mlx5e_rq_param rq_param; +}; + +void mlx5e_close_trap(struct mlx5e_trap *trap); +void mlx5e_deactivate_trap(struct mlx5e_priv *priv); +int mlx5e_handle_trap_event(struct mlx5e_priv *priv, struct mlx5_trap_ctx *trap_ctx); +#endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index bed2f1a6d730..ec5bb48cb54a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -649,8 +649,7 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq) mlx5_wq_destroy(&rq->wq_ctrl); } -static int mlx5e_create_rq(struct mlx5e_rq *rq, - struct mlx5e_rq_param *param) +int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param) { struct mlx5_core_dev *mdev = rq->mdev; @@ -773,7 +772,7 @@ static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd) return err; } -static void mlx5e_destroy_rq(struct mlx5e_rq *rq) +void mlx5e_destroy_rq(struct mlx5e_rq *rq) { mlx5_core_destroy_rq(rq->mdev, rq->rqn); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index dec93d57542f..98b56f495b32 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -52,6 +52,7 @@ #include "en/xsk/rx.h" #include "en/health.h" #include "en/params.h" +#include "devlink.h" static struct sk_buff * mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, @@ -1815,3 +1816,48 @@ int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool return 0; } + +static void mlx5e_trap_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) +{ + struct mlx5e_priv *priv = netdev_priv(rq->netdev); + struct mlx5_wq_cyc *wq = &rq->wqe.wq; + struct mlx5e_wqe_frag_info *wi; + struct sk_buff *skb; + u32 cqe_bcnt; + u16 trap_id; + u16 ci; + + trap_id = get_cqe_flow_tag(cqe); + ci = mlx5_wq_cyc_ctr2ix(wq, be16_to_cpu(cqe->wqe_counter)); + wi = get_frag(rq, ci); + cqe_bcnt = be32_to_cpu(cqe->byte_cnt); + + if (unlikely(MLX5E_RX_ERR_CQE(cqe))) { + rq->stats->wqe_err++; + goto free_wqe; + } + + skb = mlx5e_skb_from_cqe_nonlinear(rq, cqe, wi, cqe_bcnt); + if (!skb) + goto free_wqe; + + mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb); + skb_push(skb, ETH_HLEN); + + mlx5_devlink_trap_report(rq->mdev, trap_id, skb, &priv->dl_port); + dev_kfree_skb_any(skb); + +free_wqe: + mlx5e_free_rx_wqe(rq, wi, false); + mlx5_wq_cyc_pop(wq); +} + +void mlx5e_rq_set_trap_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params) +{ + rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(params, NULL) ? + mlx5e_skb_from_cqe_linear : + mlx5e_skb_from_cqe_nonlinear; + rq->post_wqes = mlx5e_post_rx_wqes; + rq->dealloc_wqe = mlx5e_dealloc_rx_wqe; + rq->handle_rx_cqe = mlx5e_trap_handle_rx_cqe; +} diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 77ba54d38772..00057eae89ab 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -903,6 +903,11 @@ static inline u64 get_cqe_ts(struct mlx5_cqe64 *cqe) return (u64)lo | ((u64)hi << 32); } +static inline u16 get_cqe_flow_tag(struct mlx5_cqe64 *cqe) +{ + return be32_to_cpu(cqe->sop_drop_qpn) & 0xFFF; +} + #define MLX5_MPWQE_LOG_NUM_STRIDES_BASE (9) #define MLX5_MPWQE_LOG_STRIDE_SZ_BASE (6) -- cgit v1.2.3 From 1ce53e2c2ac069e7b3c400a427002a70deb4a916 Mon Sep 17 00:00:00 2001 From: Alejandro Colomar Date: Sat, 28 Nov 2020 13:39:46 +0100 Subject: futex: Change utime parameter to be 'const ... *' futex(2) says that 'utime' is a pointer to 'const'. The implementation doesn't use 'const'; however, it _never_ modifies the contents of utime. - futex() either uses 'utime' as a pointer to struct or as a 'u32'. - In case it's used as a 'u32', it makes a copy of it, and of course it is not dereferenced. - In case it's used as a 'struct __kernel_timespec __user *', the pointer is not dereferenced inside the futex() definition, and it is only passed to a function: get_timespec64(), which accepts a 'const struct __kernel_timespec __user *'. [ tglx: Make the same change to the compat syscall and fixup the prototypes. ] Signed-off-by: Alejandro Colomar Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20201128123945.4592-1-alx.manpages@gmail.com --- include/linux/syscalls.h | 8 ++++---- kernel/futex.c | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index f3929aff39cf..5cb74edd9a4f 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -583,11 +583,11 @@ asmlinkage long sys_unshare(unsigned long unshare_flags); /* kernel/futex.c */ asmlinkage long sys_futex(u32 __user *uaddr, int op, u32 val, - struct __kernel_timespec __user *utime, u32 __user *uaddr2, - u32 val3); + const struct __kernel_timespec __user *utime, + u32 __user *uaddr2, u32 val3); asmlinkage long sys_futex_time32(u32 __user *uaddr, int op, u32 val, - struct old_timespec32 __user *utime, u32 __user *uaddr2, - u32 val3); + const struct old_timespec32 __user *utime, + u32 __user *uaddr2, u32 val3); asmlinkage long sys_get_robust_list(int pid, struct robust_list_head __user * __user *head_ptr, size_t __user *len_ptr); diff --git a/kernel/futex.c b/kernel/futex.c index c47d1015d759..d0775aab8da9 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -3790,8 +3790,8 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, - struct __kernel_timespec __user *, utime, u32 __user *, uaddr2, - u32, val3) + const struct __kernel_timespec __user *, utime, + u32 __user *, uaddr2, u32, val3) { struct timespec64 ts; ktime_t t, *tp = NULL; @@ -3986,7 +3986,7 @@ err_unlock: #ifdef CONFIG_COMPAT_32BIT_TIME SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val, - struct old_timespec32 __user *, utime, u32 __user *, uaddr2, + const struct old_timespec32 __user *, utime, u32 __user *, uaddr2, u32, val3) { struct timespec64 ts; -- cgit v1.2.3 From 28af22c6c8dff6a16163e5b6a56211d5b535c97b Mon Sep 17 00:00:00 2001 From: Jesper Dangaard Brouer Date: Tue, 26 Jan 2021 18:39:39 +0100 Subject: net: adjust net_device layout for cacheline usage The current layout of net_device is not optimal for cacheline usage. The member adj_list.lower linked list is split between cacheline 2 and 3. The ifindex is placed together with stats (struct net_device_stats), although most modern drivers don't update this stats member. The members netdev_ops, mtu and hard_header_len are placed on three different cachelines. These members are accessed for XDP redirect into devmap, which were noticeably with perf tool. When not using the map redirect variant (like TC-BPF does), then ifindex is also used, which is placed on a separate fourth cacheline. These members are also accessed during forwarding with regular network stack. The members priv_flags and flags are on fast-path for network stack transmit path in __dev_queue_xmit (currently located together with mtu cacheline). This patch creates a read mostly cacheline, with the purpose of keeping the above mentioned members on the same cacheline. Some netdev_features_t members also becomes part of this cacheline, which is on purpose, as function netif_skb_features() is on fast-path via validate_xmit_skb(). Signed-off-by: Jesper Dangaard Brouer Link: https://lore.kernel.org/r/161168277983.410784.12401225493601624417.stgit@firesoul Signed-off-by: Jakub Kicinski --- include/linux/netdevice.h | 53 ++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 9e8572533d8e..e9e7ada07ea1 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -1858,7 +1858,6 @@ struct net_device { unsigned long mem_end; unsigned long mem_start; unsigned long base_addr; - int irq; /* * Some hardware also needs these fields (state,dev_list, @@ -1880,6 +1879,23 @@ struct net_device { struct list_head lower; } adj_list; + /* Read-mostly cache-line for fast-path access */ + unsigned int flags; + unsigned int priv_flags; + const struct net_device_ops *netdev_ops; + int ifindex; + unsigned short gflags; + unsigned short hard_header_len; + + /* Note : dev->mtu is often read without holding a lock. + * Writers usually hold RTNL. + * It is recommended to use READ_ONCE() to annotate the reads, + * and to use WRITE_ONCE() to annotate the writes. + */ + unsigned int mtu; + unsigned short needed_headroom; + unsigned short needed_tailroom; + netdev_features_t features; netdev_features_t hw_features; netdev_features_t wanted_features; @@ -1888,10 +1904,15 @@ struct net_device { netdev_features_t mpls_features; netdev_features_t gso_partial_features; - int ifindex; + unsigned int min_mtu; + unsigned int max_mtu; + unsigned short type; + unsigned char min_header_len; + unsigned char name_assign_type; + int group; - struct net_device_stats stats; + struct net_device_stats stats; /* not used by modern drivers */ atomic_long_t rx_dropped; atomic_long_t tx_dropped; @@ -1905,7 +1926,6 @@ struct net_device { const struct iw_handler_def *wireless_handlers; struct iw_public_data *wireless_data; #endif - const struct net_device_ops *netdev_ops; const struct ethtool_ops *ethtool_ops; #ifdef CONFIG_NET_L3_MASTER_DEV const struct l3mdev_ops *l3mdev_ops; @@ -1924,34 +1944,12 @@ struct net_device { const struct header_ops *header_ops; - unsigned int flags; - unsigned int priv_flags; - - unsigned short gflags; - unsigned short padded; - unsigned char operstate; unsigned char link_mode; unsigned char if_port; unsigned char dma; - /* Note : dev->mtu is often read without holding a lock. - * Writers usually hold RTNL. - * It is recommended to use READ_ONCE() to annotate the reads, - * and to use WRITE_ONCE() to annotate the writes. - */ - unsigned int mtu; - unsigned int min_mtu; - unsigned int max_mtu; - unsigned short type; - unsigned short hard_header_len; - unsigned char min_header_len; - unsigned char name_assign_type; - - unsigned short needed_headroom; - unsigned short needed_tailroom; - /* Interface address info. */ unsigned char perm_addr[MAX_ADDR_LEN]; unsigned char addr_assign_type; @@ -1962,7 +1960,10 @@ struct net_device { unsigned short neigh_priv_len; unsigned short dev_id; unsigned short dev_port; + unsigned short padded; + spinlock_t addr_list_lock; + int irq; struct netdev_hw_addr_list uc; struct netdev_hw_addr_list mc; -- cgit v1.2.3 From f8408264c77a0cebb20244d1f4750501b36abe0e Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 14 Jan 2021 17:05:30 +0530 Subject: drivers: Remove CONFIG_OPROFILE support The "oprofile" user-space tools don't use the kernel OPROFILE support any more, and haven't in a long time. User-space has been converted to the perf interfaces. Remove kernel's old oprofile support. Suggested-by: Christoph Hellwig Suggested-by: Linus Torvalds Signed-off-by: Viresh Kumar Acked-by: Robert Richter Acked-by: Paul E. McKenney #RCU Acked-by: William Cohen Acked-by: Al Viro Acked-by: Thomas Gleixner --- Documentation/RCU/NMI-RCU.rst | 3 +- Documentation/admin-guide/kernel-parameters.txt | 14 - Documentation/process/magic-number.rst | 1 - .../translations/it_IT/process/magic-number.rst | 1 - .../translations/zh_CN/process/magic-number.rst | 1 - MAINTAINERS | 11 - arch/Kconfig | 32 -- drivers/oprofile/buffer_sync.c | 591 --------------------- drivers/oprofile/buffer_sync.h | 22 - drivers/oprofile/cpu_buffer.c | 465 ---------------- drivers/oprofile/cpu_buffer.h | 121 ----- drivers/oprofile/event_buffer.c | 209 -------- drivers/oprofile/event_buffer.h | 40 -- drivers/oprofile/nmi_timer_int.c | 157 ------ drivers/oprofile/oprof.c | 286 ---------- drivers/oprofile/oprof.h | 50 -- drivers/oprofile/oprofile_files.c | 201 ------- drivers/oprofile/oprofile_perf.c | 328 ------------ drivers/oprofile/oprofile_stats.c | 84 --- drivers/oprofile/oprofile_stats.h | 33 -- drivers/oprofile/oprofilefs.c | 300 ----------- drivers/oprofile/timer_int.c | 122 ----- include/linux/oprofile.h | 209 -------- init/Kconfig | 2 +- 24 files changed, 2 insertions(+), 3281 deletions(-) delete mode 100644 drivers/oprofile/buffer_sync.c delete mode 100644 drivers/oprofile/buffer_sync.h delete mode 100644 drivers/oprofile/cpu_buffer.c delete mode 100644 drivers/oprofile/cpu_buffer.h delete mode 100644 drivers/oprofile/event_buffer.c delete mode 100644 drivers/oprofile/event_buffer.h delete mode 100644 drivers/oprofile/nmi_timer_int.c delete mode 100644 drivers/oprofile/oprof.c delete mode 100644 drivers/oprofile/oprof.h delete mode 100644 drivers/oprofile/oprofile_files.c delete mode 100644 drivers/oprofile/oprofile_perf.c delete mode 100644 drivers/oprofile/oprofile_stats.c delete mode 100644 drivers/oprofile/oprofile_stats.h delete mode 100644 drivers/oprofile/oprofilefs.c delete mode 100644 drivers/oprofile/timer_int.c delete mode 100644 include/linux/oprofile.h (limited to 'include/linux') diff --git a/Documentation/RCU/NMI-RCU.rst b/Documentation/RCU/NMI-RCU.rst index 180958388ff9..2a92bc685ef1 100644 --- a/Documentation/RCU/NMI-RCU.rst +++ b/Documentation/RCU/NMI-RCU.rst @@ -8,8 +8,7 @@ Although RCU is usually used to protect read-mostly data structures, it is possible to use RCU to provide dynamic non-maskable interrupt handlers, as well as dynamic irq handlers. This document describes how to do this, drawing loosely from Zwane Mwaikambo's NMI-timer -work in "arch/x86/oprofile/nmi_timer_int.c" and in -"arch/x86/kernel/traps.c". +work in "arch/x86/kernel/traps.c". The relevant pieces of code are listed below, each followed by a brief explanation:: diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 9e3cdb271d06..e860c681766b 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3458,20 +3458,6 @@ For example, to override I2C bus2: omap_mux=i2c2_scl.i2c2_scl=0x100,i2c2_sda.i2c2_sda=0x100 - oprofile.timer= [HW] - Use timer interrupt instead of performance counters - - oprofile.cpu_type= Force an oprofile cpu type - This might be useful if you have an older oprofile - userland or if you want common events. - Format: { arch_perfmon } - arch_perfmon: [X86] Force use of architectural - perfmon on Intel CPUs instead of the - CPU specific event set. - timer: [X86] Force use of architectural NMI - timer mode (see also oprofile.timer - for generic hr timer mode) - oops=panic Always panic on oopses. Default is to just kill the process, but there is a small probability of deadlocking the machine. diff --git a/Documentation/process/magic-number.rst b/Documentation/process/magic-number.rst index e02ff5ffb653..c6dfe060ec2f 100644 --- a/Documentation/process/magic-number.rst +++ b/Documentation/process/magic-number.rst @@ -135,7 +135,6 @@ FW_HEADER_MAGIC 0x65726F66 fw_header ``drivers/atm/fo SLOT_MAGIC 0x67267321 slot ``drivers/hotplug/cpqphp.h`` SLOT_MAGIC 0x67267322 slot ``drivers/hotplug/acpiphp.h`` LO_MAGIC 0x68797548 nbd_device ``include/linux/nbd.h`` -OPROFILE_MAGIC 0x6f70726f super_block ``drivers/oprofile/oprofilefs.h`` M3_STATE_MAGIC 0x734d724d m3_state ``sound/oss/maestro3.c`` VMALLOC_MAGIC 0x87654320 snd_alloc_track ``sound/core/memory.c`` KMALLOC_MAGIC 0x87654321 snd_alloc_track ``sound/core/memory.c`` diff --git a/Documentation/translations/it_IT/process/magic-number.rst b/Documentation/translations/it_IT/process/magic-number.rst index 0243d32a0b59..1af30f4228f2 100644 --- a/Documentation/translations/it_IT/process/magic-number.rst +++ b/Documentation/translations/it_IT/process/magic-number.rst @@ -141,7 +141,6 @@ FW_HEADER_MAGIC 0x65726F66 fw_header ``drivers/atm/fo SLOT_MAGIC 0x67267321 slot ``drivers/hotplug/cpqphp.h`` SLOT_MAGIC 0x67267322 slot ``drivers/hotplug/acpiphp.h`` LO_MAGIC 0x68797548 nbd_device ``include/linux/nbd.h`` -OPROFILE_MAGIC 0x6f70726f super_block ``drivers/oprofile/oprofilefs.h`` M3_STATE_MAGIC 0x734d724d m3_state ``sound/oss/maestro3.c`` VMALLOC_MAGIC 0x87654320 snd_alloc_track ``sound/core/memory.c`` KMALLOC_MAGIC 0x87654321 snd_alloc_track ``sound/core/memory.c`` diff --git a/Documentation/translations/zh_CN/process/magic-number.rst b/Documentation/translations/zh_CN/process/magic-number.rst index de182bf4191c..7bb9d4165ed3 100644 --- a/Documentation/translations/zh_CN/process/magic-number.rst +++ b/Documentation/translations/zh_CN/process/magic-number.rst @@ -124,7 +124,6 @@ FW_HEADER_MAGIC 0x65726F66 fw_header ``drivers/atm/fo SLOT_MAGIC 0x67267321 slot ``drivers/hotplug/cpqphp.h`` SLOT_MAGIC 0x67267322 slot ``drivers/hotplug/acpiphp.h`` LO_MAGIC 0x68797548 nbd_device ``include/linux/nbd.h`` -OPROFILE_MAGIC 0x6f70726f super_block ``drivers/oprofile/oprofilefs.h`` M3_STATE_MAGIC 0x734d724d m3_state ``sound/oss/maestro3.c`` VMALLOC_MAGIC 0x87654320 snd_alloc_track ``sound/core/memory.c`` KMALLOC_MAGIC 0x87654321 snd_alloc_track ``sound/core/memory.c`` diff --git a/MAINTAINERS b/MAINTAINERS index cc1e6a5ee6e6..ae2c2cef9d96 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1414,7 +1414,6 @@ F: arch/arm*/include/asm/hw_breakpoint.h F: arch/arm*/include/asm/perf_event.h F: arch/arm*/kernel/hw_breakpoint.c F: arch/arm*/kernel/perf_* -F: arch/arm/oprofile/common.c F: drivers/perf/ F: include/linux/perf/arm_pmu.h @@ -4083,7 +4082,6 @@ W: http://www.ibm.com/developerworks/power/cell/ F: arch/powerpc/include/asm/cell*.h F: arch/powerpc/include/asm/spu*.h F: arch/powerpc/include/uapi/asm/spu*.h -F: arch/powerpc/oprofile/*cell* F: arch/powerpc/platforms/cell/ CELLWISE CW2015 BATTERY DRIVER @@ -13311,15 +13309,6 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git F: sound/drivers/opl4/ -OPROFILE -M: Robert Richter -L: oprofile-list@lists.sf.net -S: Maintained -F: arch/*/include/asm/oprofile*.h -F: arch/*/oprofile/ -F: drivers/oprofile/ -F: include/linux/oprofile.h - ORACLE CLUSTER FILESYSTEM 2 (OCFS2) M: Mark Fasheh M: Joel Becker diff --git a/arch/Kconfig b/arch/Kconfig index 24862d15f3a3..40277027a255 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -33,38 +33,6 @@ config HOTPLUG_SMT config GENERIC_ENTRY bool -config OPROFILE - tristate "OProfile system profiling" - depends on PROFILING - depends on HAVE_OPROFILE - select RING_BUFFER - select RING_BUFFER_ALLOW_SWAP - help - OProfile is a profiling system capable of profiling the - whole system, include the kernel, kernel modules, libraries, - and applications. - - If unsure, say N. - -config OPROFILE_EVENT_MULTIPLEX - bool "OProfile multiplexing support (EXPERIMENTAL)" - default n - depends on OPROFILE && X86 - help - The number of hardware counters is limited. The multiplexing - feature enables OProfile to gather more events than counters - are provided by the hardware. This is realized by switching - between events at a user specified time interval. - - If unsure, say N. - -config HAVE_OPROFILE - bool - -config OPROFILE_NMI_TIMER - def_bool y - depends on PERF_EVENTS && HAVE_PERF_EVENTS_NMI && !PPC64 - config KPROBES bool "Kprobes" depends on MODULES diff --git a/drivers/oprofile/buffer_sync.c b/drivers/oprofile/buffer_sync.c deleted file mode 100644 index cc917865f13a..000000000000 --- a/drivers/oprofile/buffer_sync.c +++ /dev/null @@ -1,591 +0,0 @@ -/** - * @file buffer_sync.c - * - * @remark Copyright 2002-2009 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - * @author Barry Kasindorf - * @author Robert Richter - * - * This is the core of the buffer management. Each - * CPU buffer is processed and entered into the - * global event buffer. Such processing is necessary - * in several circumstances, mentioned below. - * - * The processing does the job of converting the - * transitory EIP value into a persistent dentry/offset - * value that the profiler can record at its leisure. - * - * See fs/dcookies.c for a description of the dentry/offset - * objects. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "oprofile_stats.h" -#include "event_buffer.h" -#include "cpu_buffer.h" -#include "buffer_sync.h" - -static LIST_HEAD(dying_tasks); -static LIST_HEAD(dead_tasks); -static cpumask_var_t marked_cpus; -static DEFINE_SPINLOCK(task_mortuary); -static void process_task_mortuary(void); - -/* Take ownership of the task struct and place it on the - * list for processing. Only after two full buffer syncs - * does the task eventually get freed, because by then - * we are sure we will not reference it again. - * Can be invoked from softirq via RCU callback due to - * call_rcu() of the task struct, hence the _irqsave. - */ -static int -task_free_notify(struct notifier_block *self, unsigned long val, void *data) -{ - unsigned long flags; - struct task_struct *task = data; - spin_lock_irqsave(&task_mortuary, flags); - list_add(&task->tasks, &dying_tasks); - spin_unlock_irqrestore(&task_mortuary, flags); - return NOTIFY_OK; -} - - -/* The task is on its way out. A sync of the buffer means we can catch - * any remaining samples for this task. - */ -static int -task_exit_notify(struct notifier_block *self, unsigned long val, void *data) -{ - /* To avoid latency problems, we only process the current CPU, - * hoping that most samples for the task are on this CPU - */ - sync_buffer(raw_smp_processor_id()); - return 0; -} - - -/* The task is about to try a do_munmap(). We peek at what it's going to - * do, and if it's an executable region, process the samples first, so - * we don't lose any. This does not have to be exact, it's a QoI issue - * only. - */ -static int -munmap_notify(struct notifier_block *self, unsigned long val, void *data) -{ - unsigned long addr = (unsigned long)data; - struct mm_struct *mm = current->mm; - struct vm_area_struct *mpnt; - - mmap_read_lock(mm); - - mpnt = find_vma(mm, addr); - if (mpnt && mpnt->vm_file && (mpnt->vm_flags & VM_EXEC)) { - mmap_read_unlock(mm); - /* To avoid latency problems, we only process the current CPU, - * hoping that most samples for the task are on this CPU - */ - sync_buffer(raw_smp_processor_id()); - return 0; - } - - mmap_read_unlock(mm); - return 0; -} - - -/* We need to be told about new modules so we don't attribute to a previously - * loaded module, or drop the samples on the floor. - */ -static int -module_load_notify(struct notifier_block *self, unsigned long val, void *data) -{ -#ifdef CONFIG_MODULES - if (val != MODULE_STATE_COMING) - return NOTIFY_DONE; - - /* FIXME: should we process all CPU buffers ? */ - mutex_lock(&buffer_mutex); - add_event_entry(ESCAPE_CODE); - add_event_entry(MODULE_LOADED_CODE); - mutex_unlock(&buffer_mutex); -#endif - return NOTIFY_OK; -} - - -static struct notifier_block task_free_nb = { - .notifier_call = task_free_notify, -}; - -static struct notifier_block task_exit_nb = { - .notifier_call = task_exit_notify, -}; - -static struct notifier_block munmap_nb = { - .notifier_call = munmap_notify, -}; - -static struct notifier_block module_load_nb = { - .notifier_call = module_load_notify, -}; - -static void free_all_tasks(void) -{ - /* make sure we don't leak task structs */ - process_task_mortuary(); - process_task_mortuary(); -} - -int sync_start(void) -{ - int err; - - if (!zalloc_cpumask_var(&marked_cpus, GFP_KERNEL)) - return -ENOMEM; - - err = task_handoff_register(&task_free_nb); - if (err) - goto out1; - err = profile_event_register(PROFILE_TASK_EXIT, &task_exit_nb); - if (err) - goto out2; - err = profile_event_register(PROFILE_MUNMAP, &munmap_nb); - if (err) - goto out3; - err = register_module_notifier(&module_load_nb); - if (err) - goto out4; - - start_cpu_work(); - -out: - return err; -out4: - profile_event_unregister(PROFILE_MUNMAP, &munmap_nb); -out3: - profile_event_unregister(PROFILE_TASK_EXIT, &task_exit_nb); -out2: - task_handoff_unregister(&task_free_nb); - free_all_tasks(); -out1: - free_cpumask_var(marked_cpus); - goto out; -} - - -void sync_stop(void) -{ - end_cpu_work(); - unregister_module_notifier(&module_load_nb); - profile_event_unregister(PROFILE_MUNMAP, &munmap_nb); - profile_event_unregister(PROFILE_TASK_EXIT, &task_exit_nb); - task_handoff_unregister(&task_free_nb); - barrier(); /* do all of the above first */ - - flush_cpu_work(); - - free_all_tasks(); - free_cpumask_var(marked_cpus); -} - - -/* Optimisation. We can manage without taking the dcookie sem - * because we cannot reach this code without at least one - * dcookie user still being registered (namely, the reader - * of the event buffer). */ -static inline unsigned long fast_get_dcookie(const struct path *path) -{ - unsigned long cookie; - - if (path->dentry->d_flags & DCACHE_COOKIE) - return (unsigned long)path->dentry; - get_dcookie(path, &cookie); - return cookie; -} - - -/* Look up the dcookie for the task's mm->exe_file, - * which corresponds loosely to "application name". This is - * not strictly necessary but allows oprofile to associate - * shared-library samples with particular applications - */ -static unsigned long get_exec_dcookie(struct mm_struct *mm) -{ - unsigned long cookie = NO_COOKIE; - struct file *exe_file; - - if (!mm) - goto done; - - exe_file = get_mm_exe_file(mm); - if (!exe_file) - goto done; - - cookie = fast_get_dcookie(&exe_file->f_path); - fput(exe_file); -done: - return cookie; -} - - -/* Convert the EIP value of a sample into a persistent dentry/offset - * pair that can then be added to the global event buffer. We make - * sure to do this lookup before a mm->mmap modification happens so - * we don't lose track. - * - * The caller must ensure the mm is not nil (ie: not a kernel thread). - */ -static unsigned long -lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset) -{ - unsigned long cookie = NO_COOKIE; - struct vm_area_struct *vma; - - mmap_read_lock(mm); - for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { - - if (addr < vma->vm_start || addr >= vma->vm_end) - continue; - - if (vma->vm_file) { - cookie = fast_get_dcookie(&vma->vm_file->f_path); - *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - - vma->vm_start; - } else { - /* must be an anonymous map */ - *offset = addr; - } - - break; - } - - if (!vma) - cookie = INVALID_COOKIE; - mmap_read_unlock(mm); - - return cookie; -} - -static unsigned long last_cookie = INVALID_COOKIE; - -static void add_cpu_switch(int i) -{ - add_event_entry(ESCAPE_CODE); - add_event_entry(CPU_SWITCH_CODE); - add_event_entry(i); - last_cookie = INVALID_COOKIE; -} - -static void add_kernel_ctx_switch(unsigned int in_kernel) -{ - add_event_entry(ESCAPE_CODE); - if (in_kernel) - add_event_entry(KERNEL_ENTER_SWITCH_CODE); - else - add_event_entry(KERNEL_EXIT_SWITCH_CODE); -} - -static void -add_user_ctx_switch(struct task_struct const *task, unsigned long cookie) -{ - add_event_entry(ESCAPE_CODE); - add_event_entry(CTX_SWITCH_CODE); - add_event_entry(task->pid); - add_event_entry(cookie); - /* Another code for daemon back-compat */ - add_event_entry(ESCAPE_CODE); - add_event_entry(CTX_TGID_CODE); - add_event_entry(task->tgid); -} - - -static void add_cookie_switch(unsigned long cookie) -{ - add_event_entry(ESCAPE_CODE); - add_event_entry(COOKIE_SWITCH_CODE); - add_event_entry(cookie); -} - - -static void add_trace_begin(void) -{ - add_event_entry(ESCAPE_CODE); - add_event_entry(TRACE_BEGIN_CODE); -} - -static void add_data(struct op_entry *entry, struct mm_struct *mm) -{ - unsigned long code, pc, val; - unsigned long cookie; - off_t offset; - - if (!op_cpu_buffer_get_data(entry, &code)) - return; - if (!op_cpu_buffer_get_data(entry, &pc)) - return; - if (!op_cpu_buffer_get_size(entry)) - return; - - if (mm) { - cookie = lookup_dcookie(mm, pc, &offset); - - if (cookie == NO_COOKIE) - offset = pc; - if (cookie == INVALID_COOKIE) { - atomic_inc(&oprofile_stats.sample_lost_no_mapping); - offset = pc; - } - if (cookie != last_cookie) { - add_cookie_switch(cookie); - last_cookie = cookie; - } - } else - offset = pc; - - add_event_entry(ESCAPE_CODE); - add_event_entry(code); - add_event_entry(offset); /* Offset from Dcookie */ - - while (op_cpu_buffer_get_data(entry, &val)) - add_event_entry(val); -} - -static inline void add_sample_entry(unsigned long offset, unsigned long event) -{ - add_event_entry(offset); - add_event_entry(event); -} - - -/* - * Add a sample to the global event buffer. If possible the - * sample is converted into a persistent dentry/offset pair - * for later lookup from userspace. Return 0 on failure. - */ -static int -add_sample(struct mm_struct *mm, struct op_sample *s, int in_kernel) -{ - unsigned long cookie; - off_t offset; - - if (in_kernel) { - add_sample_entry(s->eip, s->event); - return 1; - } - - /* add userspace sample */ - - if (!mm) { - atomic_inc(&oprofile_stats.sample_lost_no_mm); - return 0; - } - - cookie = lookup_dcookie(mm, s->eip, &offset); - - if (cookie == INVALID_COOKIE) { - atomic_inc(&oprofile_stats.sample_lost_no_mapping); - return 0; - } - - if (cookie != last_cookie) { - add_cookie_switch(cookie); - last_cookie = cookie; - } - - add_sample_entry(offset, s->event); - - return 1; -} - - -static void release_mm(struct mm_struct *mm) -{ - if (!mm) - return; - mmput(mm); -} - -static inline int is_code(unsigned long val) -{ - return val == ESCAPE_CODE; -} - - -/* Move tasks along towards death. Any tasks on dead_tasks - * will definitely have no remaining references in any - * CPU buffers at this point, because we use two lists, - * and to have reached the list, it must have gone through - * one full sync already. - */ -static void process_task_mortuary(void) -{ - unsigned long flags; - LIST_HEAD(local_dead_tasks); - struct task_struct *task; - struct task_struct *ttask; - - spin_lock_irqsave(&task_mortuary, flags); - - list_splice_init(&dead_tasks, &local_dead_tasks); - list_splice_init(&dying_tasks, &dead_tasks); - - spin_unlock_irqrestore(&task_mortuary, flags); - - list_for_each_entry_safe(task, ttask, &local_dead_tasks, tasks) { - list_del(&task->tasks); - free_task(task); - } -} - - -static void mark_done(int cpu) -{ - int i; - - cpumask_set_cpu(cpu, marked_cpus); - - for_each_online_cpu(i) { - if (!cpumask_test_cpu(i, marked_cpus)) - return; - } - - /* All CPUs have been processed at least once, - * we can process the mortuary once - */ - process_task_mortuary(); - - cpumask_clear(marked_cpus); -} - - -/* FIXME: this is not sufficient if we implement syscall barrier backtrace - * traversal, the code switch to sb_sample_start at first kernel enter/exit - * switch so we need a fifth state and some special handling in sync_buffer() - */ -typedef enum { - sb_bt_ignore = -2, - sb_buffer_start, - sb_bt_start, - sb_sample_start, -} sync_buffer_state; - -/* Sync one of the CPU's buffers into the global event buffer. - * Here we need to go through each batch of samples punctuated - * by context switch notes, taking the task's mmap_lock and doing - * lookup in task->mm->mmap to convert EIP into dcookie/offset - * value. - */ -void sync_buffer(int cpu) -{ - struct mm_struct *mm = NULL; - struct mm_struct *oldmm; - unsigned long val; - struct task_struct *new; - unsigned long cookie = 0; - int in_kernel = 1; - sync_buffer_state state = sb_buffer_start; - unsigned int i; - unsigned long available; - unsigned long flags; - struct op_entry entry; - struct op_sample *sample; - - mutex_lock(&buffer_mutex); - - add_cpu_switch(cpu); - - op_cpu_buffer_reset(cpu); - available = op_cpu_buffer_entries(cpu); - - for (i = 0; i < available; ++i) { - sample = op_cpu_buffer_read_entry(&entry, cpu); - if (!sample) - break; - - if (is_code(sample->eip)) { - flags = sample->event; - if (flags & TRACE_BEGIN) { - state = sb_bt_start; - add_trace_begin(); - } - if (flags & KERNEL_CTX_SWITCH) { - /* kernel/userspace switch */ - in_kernel = flags & IS_KERNEL; - if (state == sb_buffer_start) - state = sb_sample_start; - add_kernel_ctx_switch(flags & IS_KERNEL); - } - if (flags & USER_CTX_SWITCH - && op_cpu_buffer_get_data(&entry, &val)) { - /* userspace context switch */ - new = (struct task_struct *)val; - oldmm = mm; - release_mm(oldmm); - mm = get_task_mm(new); - if (mm != oldmm) - cookie = get_exec_dcookie(mm); - add_user_ctx_switch(new, cookie); - } - if (op_cpu_buffer_get_size(&entry)) - add_data(&entry, mm); - continue; - } - - if (state < sb_bt_start) - /* ignore sample */ - continue; - - if (add_sample(mm, sample, in_kernel)) - continue; - - /* ignore backtraces if failed to add a sample */ - if (state == sb_bt_start) { - state = sb_bt_ignore; - atomic_inc(&oprofile_stats.bt_lost_no_mapping); - } - } - release_mm(mm); - - mark_done(cpu); - - mutex_unlock(&buffer_mutex); -} - -/* The function can be used to add a buffer worth of data directly to - * the kernel buffer. The buffer is assumed to be a circular buffer. - * Take the entries from index start and end at index end, wrapping - * at max_entries. - */ -void oprofile_put_buff(unsigned long *buf, unsigned int start, - unsigned int stop, unsigned int max) -{ - int i; - - i = start; - - mutex_lock(&buffer_mutex); - while (i != stop) { - add_event_entry(buf[i++]); - - if (i >= max) - i = 0; - } - - mutex_unlock(&buffer_mutex); -} - diff --git a/drivers/oprofile/buffer_sync.h b/drivers/oprofile/buffer_sync.h deleted file mode 100644 index 3110732c1835..000000000000 --- a/drivers/oprofile/buffer_sync.h +++ /dev/null @@ -1,22 +0,0 @@ -/** - * @file buffer_sync.h - * - * @remark Copyright 2002 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - */ - -#ifndef OPROFILE_BUFFER_SYNC_H -#define OPROFILE_BUFFER_SYNC_H - -/* add the necessary profiling hooks */ -int sync_start(void); - -/* remove the hooks */ -void sync_stop(void); - -/* sync the given CPU's buffer */ -void sync_buffer(int cpu); - -#endif /* OPROFILE_BUFFER_SYNC_H */ diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c deleted file mode 100644 index 9210a95cb4e6..000000000000 --- a/drivers/oprofile/cpu_buffer.c +++ /dev/null @@ -1,465 +0,0 @@ -/** - * @file cpu_buffer.c - * - * @remark Copyright 2002-2009 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - * @author Barry Kasindorf - * @author Robert Richter - * - * Each CPU has a local buffer that stores PC value/event - * pairs. We also log context switches when we notice them. - * Eventually each CPU's buffer is processed into the global - * event buffer by sync_buffer(). - * - * We use a local buffer for two reasons: an NMI or similar - * interrupt cannot synchronise, and high sampling rates - * would lead to catastrophic global synchronisation if - * a global buffer was used. - */ - -#include -#include -#include - -#include - -#include "event_buffer.h" -#include "cpu_buffer.h" -#include "buffer_sync.h" -#include "oprof.h" - -#define OP_BUFFER_FLAGS 0 - -static struct trace_buffer *op_ring_buffer; -DEFINE_PER_CPU(struct oprofile_cpu_buffer, op_cpu_buffer); - -static void wq_sync_buffer(struct work_struct *work); - -#define DEFAULT_TIMER_EXPIRE (HZ / 10) -static int work_enabled; - -unsigned long oprofile_get_cpu_buffer_size(void) -{ - return oprofile_cpu_buffer_size; -} - -void oprofile_cpu_buffer_inc_smpl_lost(void) -{ - struct oprofile_cpu_buffer *cpu_buf = this_cpu_ptr(&op_cpu_buffer); - - cpu_buf->sample_lost_overflow++; -} - -void free_cpu_buffers(void) -{ - if (op_ring_buffer) - ring_buffer_free(op_ring_buffer); - op_ring_buffer = NULL; -} - -#define RB_EVENT_HDR_SIZE 4 - -int alloc_cpu_buffers(void) -{ - int i; - - unsigned long buffer_size = oprofile_cpu_buffer_size; - unsigned long byte_size = buffer_size * (sizeof(struct op_sample) + - RB_EVENT_HDR_SIZE); - - op_ring_buffer = ring_buffer_alloc(byte_size, OP_BUFFER_FLAGS); - if (!op_ring_buffer) - goto fail; - - for_each_possible_cpu(i) { - struct oprofile_cpu_buffer *b = &per_cpu(op_cpu_buffer, i); - - b->last_task = NULL; - b->last_is_kernel = -1; - b->tracing = 0; - b->buffer_size = buffer_size; - b->sample_received = 0; - b->sample_lost_overflow = 0; - b->backtrace_aborted = 0; - b->sample_invalid_eip = 0; - b->cpu = i; - INIT_DELAYED_WORK(&b->work, wq_sync_buffer); - } - return 0; - -fail: - free_cpu_buffers(); - return -ENOMEM; -} - -void start_cpu_work(void) -{ - int i; - - work_enabled = 1; - - for_each_online_cpu(i) { - struct oprofile_cpu_buffer *b = &per_cpu(op_cpu_buffer, i); - - /* - * Spread the work by 1 jiffy per cpu so they dont all - * fire at once. - */ - schedule_delayed_work_on(i, &b->work, DEFAULT_TIMER_EXPIRE + i); - } -} - -void end_cpu_work(void) -{ - work_enabled = 0; -} - -void flush_cpu_work(void) -{ - int i; - - for_each_online_cpu(i) { - struct oprofile_cpu_buffer *b = &per_cpu(op_cpu_buffer, i); - - /* these works are per-cpu, no need for flush_sync */ - flush_delayed_work(&b->work); - } -} - -/* - * This function prepares the cpu buffer to write a sample. - * - * Struct op_entry is used during operations on the ring buffer while - * struct op_sample contains the data that is stored in the ring - * buffer. Struct entry can be uninitialized. The function reserves a - * data array that is specified by size. Use - * op_cpu_buffer_write_commit() after preparing the sample. In case of - * errors a null pointer is returned, otherwise the pointer to the - * sample. - * - */ -struct op_sample -*op_cpu_buffer_write_reserve(struct op_entry *entry, unsigned long size) -{ - entry->event = ring_buffer_lock_reserve - (op_ring_buffer, sizeof(struct op_sample) + - size * sizeof(entry->sample->data[0])); - if (!entry->event) - return NULL; - entry->sample = ring_buffer_event_data(entry->event); - entry->size = size; - entry->data = entry->sample->data; - - return entry->sample; -} - -int op_cpu_buffer_write_commit(struct op_entry *entry) -{ - return ring_buffer_unlock_commit(op_ring_buffer, entry->event); -} - -struct op_sample *op_cpu_buffer_read_entry(struct op_entry *entry, int cpu) -{ - struct ring_buffer_event *e; - e = ring_buffer_consume(op_ring_buffer, cpu, NULL, NULL); - if (!e) - return NULL; - - entry->event = e; - entry->sample = ring_buffer_event_data(e); - entry->size = (ring_buffer_event_length(e) - sizeof(struct op_sample)) - / sizeof(entry->sample->data[0]); - entry->data = entry->sample->data; - return entry->sample; -} - -unsigned long op_cpu_buffer_entries(int cpu) -{ - return ring_buffer_entries_cpu(op_ring_buffer, cpu); -} - -static int -op_add_code(struct oprofile_cpu_buffer *cpu_buf, unsigned long backtrace, - int is_kernel, struct task_struct *task) -{ - struct op_entry entry; - struct op_sample *sample; - unsigned long flags; - int size; - - flags = 0; - - if (backtrace) - flags |= TRACE_BEGIN; - - /* notice a switch from user->kernel or vice versa */ - is_kernel = !!is_kernel; - if (cpu_buf->last_is_kernel != is_kernel) { - cpu_buf->last_is_kernel = is_kernel; - flags |= KERNEL_CTX_SWITCH; - if (is_kernel) - flags |= IS_KERNEL; - } - - /* notice a task switch */ - if (cpu_buf->last_task != task) { - cpu_buf->last_task = task; - flags |= USER_CTX_SWITCH; - } - - if (!flags) - /* nothing to do */ - return 0; - - if (flags & USER_CTX_SWITCH) - size = 1; - else - size = 0; - - sample = op_cpu_buffer_write_reserve(&entry, size); - if (!sample) - return -ENOMEM; - - sample->eip = ESCAPE_CODE; - sample->event = flags; - - if (size) - op_cpu_buffer_add_data(&entry, (unsigned long)task); - - op_cpu_buffer_write_commit(&entry); - - return 0; -} - -static inline int -op_add_sample(struct oprofile_cpu_buffer *cpu_buf, - unsigned long pc, unsigned long event) -{ - struct op_entry entry; - struct op_sample *sample; - - sample = op_cpu_buffer_write_reserve(&entry, 0); - if (!sample) - return -ENOMEM; - - sample->eip = pc; - sample->event = event; - - return op_cpu_buffer_write_commit(&entry); -} - -/* - * This must be safe from any context. - * - * is_kernel is needed because on some architectures you cannot - * tell if you are in kernel or user space simply by looking at - * pc. We tag this in the buffer by generating kernel enter/exit - * events whenever is_kernel changes - */ -static int -log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc, - unsigned long backtrace, int is_kernel, unsigned long event, - struct task_struct *task) -{ - struct task_struct *tsk = task ? task : current; - cpu_buf->sample_received++; - - if (pc == ESCAPE_CODE) { - cpu_buf->sample_invalid_eip++; - return 0; - } - - if (op_add_code(cpu_buf, backtrace, is_kernel, tsk)) - goto fail; - - if (op_add_sample(cpu_buf, pc, event)) - goto fail; - - return 1; - -fail: - cpu_buf->sample_lost_overflow++; - return 0; -} - -static inline void oprofile_begin_trace(struct oprofile_cpu_buffer *cpu_buf) -{ - cpu_buf->tracing = 1; -} - -static inline void oprofile_end_trace(struct oprofile_cpu_buffer *cpu_buf) -{ - cpu_buf->tracing = 0; -} - -static inline void -__oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs, - unsigned long event, int is_kernel, - struct task_struct *task) -{ - struct oprofile_cpu_buffer *cpu_buf = this_cpu_ptr(&op_cpu_buffer); - unsigned long backtrace = oprofile_backtrace_depth; - - /* - * if log_sample() fail we can't backtrace since we lost the - * source of this event - */ - if (!log_sample(cpu_buf, pc, backtrace, is_kernel, event, task)) - /* failed */ - return; - - if (!backtrace) - return; - - oprofile_begin_trace(cpu_buf); - oprofile_ops.backtrace(regs, backtrace); - oprofile_end_trace(cpu_buf); -} - -void oprofile_add_ext_hw_sample(unsigned long pc, struct pt_regs * const regs, - unsigned long event, int is_kernel, - struct task_struct *task) -{ - __oprofile_add_ext_sample(pc, regs, event, is_kernel, task); -} - -void oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs, - unsigned long event, int is_kernel) -{ - __oprofile_add_ext_sample(pc, regs, event, is_kernel, NULL); -} - -void oprofile_add_sample(struct pt_regs * const regs, unsigned long event) -{ - int is_kernel; - unsigned long pc; - - if (likely(regs)) { - is_kernel = !user_mode(regs); - pc = profile_pc(regs); - } else { - is_kernel = 0; /* This value will not be used */ - pc = ESCAPE_CODE; /* as this causes an early return. */ - } - - __oprofile_add_ext_sample(pc, regs, event, is_kernel, NULL); -} - -/* - * Add samples with data to the ring buffer. - * - * Use oprofile_add_data(&entry, val) to add data and - * oprofile_write_commit(&entry) to commit the sample. - */ -void -oprofile_write_reserve(struct op_entry *entry, struct pt_regs * const regs, - unsigned long pc, int code, int size) -{ - struct op_sample *sample; - int is_kernel = !user_mode(regs); - struct oprofile_cpu_buffer *cpu_buf = this_cpu_ptr(&op_cpu_buffer); - - cpu_buf->sample_received++; - - /* no backtraces for samples with data */ - if (op_add_code(cpu_buf, 0, is_kernel, current)) - goto fail; - - sample = op_cpu_buffer_write_reserve(entry, size + 2); - if (!sample) - goto fail; - sample->eip = ESCAPE_CODE; - sample->event = 0; /* no flags */ - - op_cpu_buffer_add_data(entry, code); - op_cpu_buffer_add_data(entry, pc); - - return; - -fail: - entry->event = NULL; - cpu_buf->sample_lost_overflow++; -} - -int oprofile_add_data(struct op_entry *entry, unsigned long val) -{ - if (!entry->event) - return 0; - return op_cpu_buffer_add_data(entry, val); -} - -int oprofile_add_data64(struct op_entry *entry, u64 val) -{ - if (!entry->event) - return 0; - if (op_cpu_buffer_get_size(entry) < 2) - /* - * the function returns 0 to indicate a too small - * buffer, even if there is some space left - */ - return 0; - if (!op_cpu_buffer_add_data(entry, (u32)val)) - return 0; - return op_cpu_buffer_add_data(entry, (u32)(val >> 32)); -} - -int oprofile_write_commit(struct op_entry *entry) -{ - if (!entry->event) - return -EINVAL; - return op_cpu_buffer_write_commit(entry); -} - -void oprofile_add_pc(unsigned long pc, int is_kernel, unsigned long event) -{ - struct oprofile_cpu_buffer *cpu_buf = this_cpu_ptr(&op_cpu_buffer); - log_sample(cpu_buf, pc, 0, is_kernel, event, NULL); -} - -void oprofile_add_trace(unsigned long pc) -{ - struct oprofile_cpu_buffer *cpu_buf = this_cpu_ptr(&op_cpu_buffer); - - if (!cpu_buf->tracing) - return; - - /* - * broken frame can give an eip with the same value as an - * escape code, abort the trace if we get it - */ - if (pc == ESCAPE_CODE) - goto fail; - - if (op_add_sample(cpu_buf, pc, 0)) - goto fail; - - return; -fail: - cpu_buf->tracing = 0; - cpu_buf->backtrace_aborted++; - return; -} - -/* - * This serves to avoid cpu buffer overflow, and makes sure - * the task mortuary progresses - * - * By using schedule_delayed_work_on and then schedule_delayed_work - * we guarantee this will stay on the correct cpu - */ -static void wq_sync_buffer(struct work_struct *work) -{ - struct oprofile_cpu_buffer *b = - container_of(work, struct oprofile_cpu_buffer, work.work); - if (b->cpu != smp_processor_id() && !cpu_online(b->cpu)) { - cancel_delayed_work(&b->work); - return; - } - sync_buffer(b->cpu); - - /* don't re-add the work if we're shutting down */ - if (work_enabled) - schedule_delayed_work(&b->work, DEFAULT_TIMER_EXPIRE); -} diff --git a/drivers/oprofile/cpu_buffer.h b/drivers/oprofile/cpu_buffer.h deleted file mode 100644 index 31478c0cff87..000000000000 --- a/drivers/oprofile/cpu_buffer.h +++ /dev/null @@ -1,121 +0,0 @@ -/** - * @file cpu_buffer.h - * - * @remark Copyright 2002-2009 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - * @author Robert Richter - */ - -#ifndef OPROFILE_CPU_BUFFER_H -#define OPROFILE_CPU_BUFFER_H - -#include -#include -#include -#include -#include -#include - -struct task_struct; - -int alloc_cpu_buffers(void); -void free_cpu_buffers(void); - -void start_cpu_work(void); -void end_cpu_work(void); -void flush_cpu_work(void); - -/* CPU buffer is composed of such entries (which are - * also used for context switch notes) - */ -struct op_sample { - unsigned long eip; - unsigned long event; - unsigned long data[]; -}; - -struct op_entry; - -struct oprofile_cpu_buffer { - unsigned long buffer_size; - struct task_struct *last_task; - int last_is_kernel; - int tracing; - unsigned long sample_received; - unsigned long sample_lost_overflow; - unsigned long backtrace_aborted; - unsigned long sample_invalid_eip; - int cpu; - struct delayed_work work; -}; - -DECLARE_PER_CPU(struct oprofile_cpu_buffer, op_cpu_buffer); - -/* - * Resets the cpu buffer to a sane state. - * - * reset these to invalid values; the next sample collected will - * populate the buffer with proper values to initialize the buffer - */ -static inline void op_cpu_buffer_reset(int cpu) -{ - struct oprofile_cpu_buffer *cpu_buf = &per_cpu(op_cpu_buffer, cpu); - - cpu_buf->last_is_kernel = -1; - cpu_buf->last_task = NULL; -} - -/* - * op_cpu_buffer_add_data() and op_cpu_buffer_write_commit() may be - * called only if op_cpu_buffer_write_reserve() did not return NULL or - * entry->event != NULL, otherwise entry->size or entry->event will be - * used uninitialized. - */ - -struct op_sample -*op_cpu_buffer_write_reserve(struct op_entry *entry, unsigned long size); -int op_cpu_buffer_write_commit(struct op_entry *entry); -struct op_sample *op_cpu_buffer_read_entry(struct op_entry *entry, int cpu); -unsigned long op_cpu_buffer_entries(int cpu); - -/* returns the remaining free size of data in the entry */ -static inline -int op_cpu_buffer_add_data(struct op_entry *entry, unsigned long val) -{ - if (!entry->size) - return 0; - *entry->data = val; - entry->size--; - entry->data++; - return entry->size; -} - -/* returns the size of data in the entry */ -static inline -int op_cpu_buffer_get_size(struct op_entry *entry) -{ - return entry->size; -} - -/* returns 0 if empty or the size of data including the current value */ -static inline -int op_cpu_buffer_get_data(struct op_entry *entry, unsigned long *val) -{ - int size = entry->size; - if (!size) - return 0; - *val = *entry->data; - entry->size--; - entry->data++; - return size; -} - -/* extra data flags */ -#define KERNEL_CTX_SWITCH (1UL << 0) -#define IS_KERNEL (1UL << 1) -#define TRACE_BEGIN (1UL << 2) -#define USER_CTX_SWITCH (1UL << 3) - -#endif /* OPROFILE_CPU_BUFFER_H */ diff --git a/drivers/oprofile/event_buffer.c b/drivers/oprofile/event_buffer.c deleted file mode 100644 index 6c9edc8bbc95..000000000000 --- a/drivers/oprofile/event_buffer.c +++ /dev/null @@ -1,209 +0,0 @@ -/** - * @file event_buffer.c - * - * @remark Copyright 2002 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - * - * This is the global event buffer that the user-space - * daemon reads from. The event buffer is an untyped array - * of unsigned longs. Entries are prefixed by the - * escape value ESCAPE_CODE followed by an identifying code. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "oprof.h" -#include "event_buffer.h" -#include "oprofile_stats.h" - -DEFINE_MUTEX(buffer_mutex); - -static unsigned long buffer_opened; -static DECLARE_WAIT_QUEUE_HEAD(buffer_wait); -static unsigned long *event_buffer; -static unsigned long buffer_size; -static unsigned long buffer_watershed; -static size_t buffer_pos; -/* atomic_t because wait_event checks it outside of buffer_mutex */ -static atomic_t buffer_ready = ATOMIC_INIT(0); - -/* - * Add an entry to the event buffer. When we get near to the end we - * wake up the process sleeping on the read() of the file. To protect - * the event_buffer this function may only be called when buffer_mutex - * is set. - */ -void add_event_entry(unsigned long value) -{ - /* - * This shouldn't happen since all workqueues or handlers are - * canceled or flushed before the event buffer is freed. - */ - if (!event_buffer) { - WARN_ON_ONCE(1); - return; - } - - if (buffer_pos == buffer_size) { - atomic_inc(&oprofile_stats.event_lost_overflow); - return; - } - - event_buffer[buffer_pos] = value; - if (++buffer_pos == buffer_size - buffer_watershed) { - atomic_set(&buffer_ready, 1); - wake_up(&buffer_wait); - } -} - - -/* Wake up the waiting process if any. This happens - * on "echo 0 >/dev/oprofile/enable" so the daemon - * processes the data remaining in the event buffer. - */ -void wake_up_buffer_waiter(void) -{ - mutex_lock(&buffer_mutex); - atomic_set(&buffer_ready, 1); - wake_up(&buffer_wait); - mutex_unlock(&buffer_mutex); -} - - -int alloc_event_buffer(void) -{ - unsigned long flags; - - raw_spin_lock_irqsave(&oprofilefs_lock, flags); - buffer_size = oprofile_buffer_size; - buffer_watershed = oprofile_buffer_watershed; - raw_spin_unlock_irqrestore(&oprofilefs_lock, flags); - - if (buffer_watershed >= buffer_size) - return -EINVAL; - - buffer_pos = 0; - event_buffer = vmalloc(array_size(buffer_size, sizeof(unsigned long))); - if (!event_buffer) - return -ENOMEM; - - return 0; -} - - -void free_event_buffer(void) -{ - mutex_lock(&buffer_mutex); - vfree(event_buffer); - buffer_pos = 0; - event_buffer = NULL; - mutex_unlock(&buffer_mutex); -} - - -static int event_buffer_open(struct inode *inode, struct file *file) -{ - int err = -EPERM; - - if (!perfmon_capable()) - return -EPERM; - - if (test_and_set_bit_lock(0, &buffer_opened)) - return -EBUSY; - - /* Register as a user of dcookies - * to ensure they persist for the lifetime of - * the open event file - */ - err = -EINVAL; - file->private_data = dcookie_register(); - if (!file->private_data) - goto out; - - if ((err = oprofile_setup())) - goto fail; - - /* NB: the actual start happens from userspace - * echo 1 >/dev/oprofile/enable - */ - - return nonseekable_open(inode, file); - -fail: - dcookie_unregister(file->private_data); -out: - __clear_bit_unlock(0, &buffer_opened); - return err; -} - - -static int event_buffer_release(struct inode *inode, struct file *file) -{ - oprofile_stop(); - oprofile_shutdown(); - dcookie_unregister(file->private_data); - buffer_pos = 0; - atomic_set(&buffer_ready, 0); - __clear_bit_unlock(0, &buffer_opened); - return 0; -} - - -static ssize_t event_buffer_read(struct file *file, char __user *buf, - size_t count, loff_t *offset) -{ - int retval = -EINVAL; - size_t const max = buffer_size * sizeof(unsigned long); - - /* handling partial reads is more trouble than it's worth */ - if (count != max || *offset) - return -EINVAL; - - wait_event_interruptible(buffer_wait, atomic_read(&buffer_ready)); - - if (signal_pending(current)) - return -EINTR; - - /* can't currently happen */ - if (!atomic_read(&buffer_ready)) - return -EAGAIN; - - mutex_lock(&buffer_mutex); - - /* May happen if the buffer is freed during pending reads. */ - if (!event_buffer) { - retval = -EINTR; - goto out; - } - - atomic_set(&buffer_ready, 0); - - retval = -EFAULT; - - count = buffer_pos * sizeof(unsigned long); - - if (copy_to_user(buf, event_buffer, count)) - goto out; - - retval = count; - buffer_pos = 0; - -out: - mutex_unlock(&buffer_mutex); - return retval; -} - -const struct file_operations event_buffer_fops = { - .open = event_buffer_open, - .release = event_buffer_release, - .read = event_buffer_read, - .llseek = no_llseek, -}; diff --git a/drivers/oprofile/event_buffer.h b/drivers/oprofile/event_buffer.h deleted file mode 100644 index a8d5bb3cba89..000000000000 --- a/drivers/oprofile/event_buffer.h +++ /dev/null @@ -1,40 +0,0 @@ -/** - * @file event_buffer.h - * - * @remark Copyright 2002 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - */ - -#ifndef EVENT_BUFFER_H -#define EVENT_BUFFER_H - -#include -#include - -int alloc_event_buffer(void); - -void free_event_buffer(void); - -/** - * Add data to the event buffer. - * The data passed is free-form, but typically consists of - * file offsets, dcookies, context information, and ESCAPE codes. - */ -void add_event_entry(unsigned long data); - -/* wake up the process sleeping on the event file */ -void wake_up_buffer_waiter(void); - -#define INVALID_COOKIE ~0UL -#define NO_COOKIE 0UL - -extern const struct file_operations event_buffer_fops; - -/* mutex between sync_cpu_buffers() and the - * file reading code. - */ -extern struct mutex buffer_mutex; - -#endif /* EVENT_BUFFER_H */ diff --git a/drivers/oprofile/nmi_timer_int.c b/drivers/oprofile/nmi_timer_int.c deleted file mode 100644 index f343bd96609a..000000000000 --- a/drivers/oprofile/nmi_timer_int.c +++ /dev/null @@ -1,157 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/** - * @file nmi_timer_int.c - * - * @remark Copyright 2011 Advanced Micro Devices, Inc. - * - * @author Robert Richter - */ - -#include -#include -#include -#include -#include - -#ifdef CONFIG_OPROFILE_NMI_TIMER - -static DEFINE_PER_CPU(struct perf_event *, nmi_timer_events); -static int ctr_running; - -static struct perf_event_attr nmi_timer_attr = { - .type = PERF_TYPE_HARDWARE, - .config = PERF_COUNT_HW_CPU_CYCLES, - .size = sizeof(struct perf_event_attr), - .pinned = 1, - .disabled = 1, -}; - -static void nmi_timer_callback(struct perf_event *event, - struct perf_sample_data *data, - struct pt_regs *regs) -{ - event->hw.interrupts = 0; /* don't throttle interrupts */ - oprofile_add_sample(regs, 0); -} - -static int nmi_timer_start_cpu(int cpu) -{ - struct perf_event *event = per_cpu(nmi_timer_events, cpu); - - if (!event) { - event = perf_event_create_kernel_counter(&nmi_timer_attr, cpu, NULL, - nmi_timer_callback, NULL); - if (IS_ERR(event)) - return PTR_ERR(event); - per_cpu(nmi_timer_events, cpu) = event; - } - - if (event && ctr_running) - perf_event_enable(event); - - return 0; -} - -static void nmi_timer_stop_cpu(int cpu) -{ - struct perf_event *event = per_cpu(nmi_timer_events, cpu); - - if (event && ctr_running) - perf_event_disable(event); -} - -static int nmi_timer_cpu_online(unsigned int cpu) -{ - nmi_timer_start_cpu(cpu); - return 0; -} -static int nmi_timer_cpu_predown(unsigned int cpu) -{ - nmi_timer_stop_cpu(cpu); - return 0; -} - -static int nmi_timer_start(void) -{ - int cpu; - - get_online_cpus(); - ctr_running = 1; - for_each_online_cpu(cpu) - nmi_timer_start_cpu(cpu); - put_online_cpus(); - - return 0; -} - -static void nmi_timer_stop(void) -{ - int cpu; - - get_online_cpus(); - for_each_online_cpu(cpu) - nmi_timer_stop_cpu(cpu); - ctr_running = 0; - put_online_cpus(); -} - -static enum cpuhp_state hp_online; - -static void nmi_timer_shutdown(void) -{ - struct perf_event *event; - int cpu; - - cpuhp_remove_state(hp_online); - for_each_possible_cpu(cpu) { - event = per_cpu(nmi_timer_events, cpu); - if (!event) - continue; - perf_event_disable(event); - per_cpu(nmi_timer_events, cpu) = NULL; - perf_event_release_kernel(event); - } -} - -static int nmi_timer_setup(void) -{ - int err; - u64 period; - - /* clock cycles per tick: */ - period = (u64)cpu_khz * 1000; - do_div(period, HZ); - nmi_timer_attr.sample_period = period; - - err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "oprofile/nmi:online", - nmi_timer_cpu_online, nmi_timer_cpu_predown); - if (err < 0) { - nmi_timer_shutdown(); - return err; - } - hp_online = err; - return 0; -} - -int __init op_nmi_timer_init(struct oprofile_operations *ops) -{ - int err = 0; - - err = nmi_timer_setup(); - if (err) - return err; - nmi_timer_shutdown(); /* only check, don't alloc */ - - ops->create_files = NULL; - ops->setup = nmi_timer_setup; - ops->shutdown = nmi_timer_shutdown; - ops->start = nmi_timer_start; - ops->stop = nmi_timer_stop; - ops->cpu_type = "timer"; - - printk(KERN_INFO "oprofile: using NMI timer interrupt.\n"); - - return 0; -} - -#endif diff --git a/drivers/oprofile/oprof.c b/drivers/oprofile/oprof.c deleted file mode 100644 index ed2c3ec07024..000000000000 --- a/drivers/oprofile/oprof.c +++ /dev/null @@ -1,286 +0,0 @@ -/** - * @file oprof.c - * - * @remark Copyright 2002 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "oprof.h" -#include "event_buffer.h" -#include "cpu_buffer.h" -#include "buffer_sync.h" -#include "oprofile_stats.h" - -struct oprofile_operations oprofile_ops; - -unsigned long oprofile_started; -unsigned long oprofile_backtrace_depth; -static unsigned long is_setup; -static DEFINE_MUTEX(start_mutex); - -/* timer - 0 - use performance monitoring hardware if available - 1 - use the timer int mechanism regardless - */ -static int timer = 0; - -int oprofile_setup(void) -{ - int err; - - mutex_lock(&start_mutex); - - if ((err = alloc_cpu_buffers())) - goto out; - - if ((err = alloc_event_buffer())) - goto out1; - - if (oprofile_ops.setup && (err = oprofile_ops.setup())) - goto out2; - - /* Note even though this starts part of the - * profiling overhead, it's necessary to prevent - * us missing task deaths and eventually oopsing - * when trying to process the event buffer. - */ - if (oprofile_ops.sync_start) { - int sync_ret = oprofile_ops.sync_start(); - switch (sync_ret) { - case 0: - goto post_sync; - case 1: - goto do_generic; - case -1: - goto out3; - default: - goto out3; - } - } -do_generic: - if ((err = sync_start())) - goto out3; - -post_sync: - is_setup = 1; - mutex_unlock(&start_mutex); - return 0; - -out3: - if (oprofile_ops.shutdown) - oprofile_ops.shutdown(); -out2: - free_event_buffer(); -out1: - free_cpu_buffers(); -out: - mutex_unlock(&start_mutex); - return err; -} - -#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX - -static void switch_worker(struct work_struct *work); -static DECLARE_DELAYED_WORK(switch_work, switch_worker); - -static void start_switch_worker(void) -{ - if (oprofile_ops.switch_events) - schedule_delayed_work(&switch_work, oprofile_time_slice); -} - -static void stop_switch_worker(void) -{ - cancel_delayed_work_sync(&switch_work); -} - -static void switch_worker(struct work_struct *work) -{ - if (oprofile_ops.switch_events()) - return; - - atomic_inc(&oprofile_stats.multiplex_counter); - start_switch_worker(); -} - -/* User inputs in ms, converts to jiffies */ -int oprofile_set_timeout(unsigned long val_msec) -{ - int err = 0; - unsigned long time_slice; - - mutex_lock(&start_mutex); - - if (oprofile_started) { - err = -EBUSY; - goto out; - } - - if (!oprofile_ops.switch_events) { - err = -EINVAL; - goto out; - } - - time_slice = msecs_to_jiffies(val_msec); - if (time_slice == MAX_JIFFY_OFFSET) { - err = -EINVAL; - goto out; - } - - oprofile_time_slice = time_slice; - -out: - mutex_unlock(&start_mutex); - return err; - -} - -#else - -static inline void start_switch_worker(void) { } -static inline void stop_switch_worker(void) { } - -#endif - -/* Actually start profiling (echo 1>/dev/oprofile/enable) */ -int oprofile_start(void) -{ - int err = -EINVAL; - - mutex_lock(&start_mutex); - - if (!is_setup) - goto out; - - err = 0; - - if (oprofile_started) - goto out; - - oprofile_reset_stats(); - - if ((err = oprofile_ops.start())) - goto out; - - start_switch_worker(); - - oprofile_started = 1; -out: - mutex_unlock(&start_mutex); - return err; -} - - -/* echo 0>/dev/oprofile/enable */ -void oprofile_stop(void) -{ - mutex_lock(&start_mutex); - if (!oprofile_started) - goto out; - oprofile_ops.stop(); - oprofile_started = 0; - - stop_switch_worker(); - - /* wake up the daemon to read what remains */ - wake_up_buffer_waiter(); -out: - mutex_unlock(&start_mutex); -} - - -void oprofile_shutdown(void) -{ - mutex_lock(&start_mutex); - if (oprofile_ops.sync_stop) { - int sync_ret = oprofile_ops.sync_stop(); - switch (sync_ret) { - case 0: - goto post_sync; - case 1: - goto do_generic; - default: - goto post_sync; - } - } -do_generic: - sync_stop(); -post_sync: - if (oprofile_ops.shutdown) - oprofile_ops.shutdown(); - is_setup = 0; - free_event_buffer(); - free_cpu_buffers(); - mutex_unlock(&start_mutex); -} - -int oprofile_set_ulong(unsigned long *addr, unsigned long val) -{ - int err = -EBUSY; - - mutex_lock(&start_mutex); - if (!oprofile_started) { - *addr = val; - err = 0; - } - mutex_unlock(&start_mutex); - - return err; -} - -static int timer_mode; - -static int __init oprofile_init(void) -{ - int err; - - /* always init architecture to setup backtrace support */ - timer_mode = 0; - err = oprofile_arch_init(&oprofile_ops); - if (!err) { - if (!timer && !oprofilefs_register()) - return 0; - oprofile_arch_exit(); - } - - /* setup timer mode: */ - timer_mode = 1; - /* no nmi timer mode if oprofile.timer is set */ - if (timer || op_nmi_timer_init(&oprofile_ops)) { - err = oprofile_timer_init(&oprofile_ops); - if (err) - return err; - } - - return oprofilefs_register(); -} - - -static void __exit oprofile_exit(void) -{ - oprofilefs_unregister(); - if (!timer_mode) - oprofile_arch_exit(); -} - - -module_init(oprofile_init); -module_exit(oprofile_exit); - -module_param_named(timer, timer, int, 0644); -MODULE_PARM_DESC(timer, "force use of timer interrupt"); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("John Levon "); -MODULE_DESCRIPTION("OProfile system profiler"); diff --git a/drivers/oprofile/oprof.h b/drivers/oprofile/oprof.h deleted file mode 100644 index d5412060ab0f..000000000000 --- a/drivers/oprofile/oprof.h +++ /dev/null @@ -1,50 +0,0 @@ -/** - * @file oprof.h - * - * @remark Copyright 2002 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - */ - -#ifndef OPROF_H -#define OPROF_H - -int oprofile_setup(void); -void oprofile_shutdown(void); - -int oprofilefs_register(void); -void oprofilefs_unregister(void); - -int oprofile_start(void); -void oprofile_stop(void); - -struct oprofile_operations; - -extern unsigned long oprofile_buffer_size; -extern unsigned long oprofile_cpu_buffer_size; -extern unsigned long oprofile_buffer_watershed; -extern unsigned long oprofile_time_slice; - -extern struct oprofile_operations oprofile_ops; -extern unsigned long oprofile_started; -extern unsigned long oprofile_backtrace_depth; - -struct dentry; - -void oprofile_create_files(struct dentry *root); -int oprofile_timer_init(struct oprofile_operations *ops); -#ifdef CONFIG_OPROFILE_NMI_TIMER -int op_nmi_timer_init(struct oprofile_operations *ops); -#else -static inline int op_nmi_timer_init(struct oprofile_operations *ops) -{ - return -ENODEV; -} -#endif - - -int oprofile_set_ulong(unsigned long *addr, unsigned long val); -int oprofile_set_timeout(unsigned long time); - -#endif /* OPROF_H */ diff --git a/drivers/oprofile/oprofile_files.c b/drivers/oprofile/oprofile_files.c deleted file mode 100644 index ee2cfce358b9..000000000000 --- a/drivers/oprofile/oprofile_files.c +++ /dev/null @@ -1,201 +0,0 @@ -/** - * @file oprofile_files.c - * - * @remark Copyright 2002 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - */ - -#include -#include -#include - -#include "event_buffer.h" -#include "oprofile_stats.h" -#include "oprof.h" - -#define BUFFER_SIZE_DEFAULT 131072 -#define CPU_BUFFER_SIZE_DEFAULT 8192 -#define BUFFER_WATERSHED_DEFAULT 32768 /* FIXME: tune */ -#define TIME_SLICE_DEFAULT 1 - -unsigned long oprofile_buffer_size; -unsigned long oprofile_cpu_buffer_size; -unsigned long oprofile_buffer_watershed; -unsigned long oprofile_time_slice; - -#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX - -static ssize_t timeout_read(struct file *file, char __user *buf, - size_t count, loff_t *offset) -{ - return oprofilefs_ulong_to_user(jiffies_to_msecs(oprofile_time_slice), - buf, count, offset); -} - - -static ssize_t timeout_write(struct file *file, char const __user *buf, - size_t count, loff_t *offset) -{ - unsigned long val; - int retval; - - if (*offset) - return -EINVAL; - - retval = oprofilefs_ulong_from_user(&val, buf, count); - if (retval <= 0) - return retval; - - retval = oprofile_set_timeout(val); - - if (retval) - return retval; - return count; -} - - -static const struct file_operations timeout_fops = { - .read = timeout_read, - .write = timeout_write, - .llseek = default_llseek, -}; - -#endif - - -static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset) -{ - return oprofilefs_ulong_to_user(oprofile_backtrace_depth, buf, count, - offset); -} - - -static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) -{ - unsigned long val; - int retval; - - if (*offset) - return -EINVAL; - - if (!oprofile_ops.backtrace) - return -EINVAL; - - retval = oprofilefs_ulong_from_user(&val, buf, count); - if (retval <= 0) - return retval; - - retval = oprofile_set_ulong(&oprofile_backtrace_depth, val); - if (retval) - return retval; - - return count; -} - - -static const struct file_operations depth_fops = { - .read = depth_read, - .write = depth_write, - .llseek = default_llseek, -}; - - -static ssize_t pointer_size_read(struct file *file, char __user *buf, size_t count, loff_t *offset) -{ - return oprofilefs_ulong_to_user(sizeof(void *), buf, count, offset); -} - - -static const struct file_operations pointer_size_fops = { - .read = pointer_size_read, - .llseek = default_llseek, -}; - - -static ssize_t cpu_type_read(struct file *file, char __user *buf, size_t count, loff_t *offset) -{ - return oprofilefs_str_to_user(oprofile_ops.cpu_type, buf, count, offset); -} - - -static const struct file_operations cpu_type_fops = { - .read = cpu_type_read, - .llseek = default_llseek, -}; - - -static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset) -{ - return oprofilefs_ulong_to_user(oprofile_started, buf, count, offset); -} - - -static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) -{ - unsigned long val; - int retval; - - if (*offset) - return -EINVAL; - - retval = oprofilefs_ulong_from_user(&val, buf, count); - if (retval <= 0) - return retval; - - retval = 0; - if (val) - retval = oprofile_start(); - else - oprofile_stop(); - - if (retval) - return retval; - return count; -} - - -static const struct file_operations enable_fops = { - .read = enable_read, - .write = enable_write, - .llseek = default_llseek, -}; - - -static ssize_t dump_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) -{ - wake_up_buffer_waiter(); - return count; -} - - -static const struct file_operations dump_fops = { - .write = dump_write, - .llseek = noop_llseek, -}; - -void oprofile_create_files(struct dentry *root) -{ - /* reinitialize default values */ - oprofile_buffer_size = BUFFER_SIZE_DEFAULT; - oprofile_cpu_buffer_size = CPU_BUFFER_SIZE_DEFAULT; - oprofile_buffer_watershed = BUFFER_WATERSHED_DEFAULT; - oprofile_time_slice = msecs_to_jiffies(TIME_SLICE_DEFAULT); - - oprofilefs_create_file(root, "enable", &enable_fops); - oprofilefs_create_file_perm(root, "dump", &dump_fops, 0666); - oprofilefs_create_file(root, "buffer", &event_buffer_fops); - oprofilefs_create_ulong(root, "buffer_size", &oprofile_buffer_size); - oprofilefs_create_ulong(root, "buffer_watershed", &oprofile_buffer_watershed); - oprofilefs_create_ulong(root, "cpu_buffer_size", &oprofile_cpu_buffer_size); - oprofilefs_create_file(root, "cpu_type", &cpu_type_fops); - oprofilefs_create_file(root, "backtrace_depth", &depth_fops); - oprofilefs_create_file(root, "pointer_size", &pointer_size_fops); -#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX - oprofilefs_create_file(root, "time_slice", &timeout_fops); -#endif - oprofile_create_stats_files(root); - if (oprofile_ops.create_files) - oprofile_ops.create_files(root); -} diff --git a/drivers/oprofile/oprofile_perf.c b/drivers/oprofile/oprofile_perf.c deleted file mode 100644 index 98a63a5f8763..000000000000 --- a/drivers/oprofile/oprofile_perf.c +++ /dev/null @@ -1,328 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright 2010 ARM Ltd. - * Copyright 2012 Advanced Micro Devices, Inc., Robert Richter - * - * Perf-events backend for OProfile. - */ -#include -#include -#include -#include - -/* - * Per performance monitor configuration as set via oprofilefs. - */ -struct op_counter_config { - unsigned long count; - unsigned long enabled; - unsigned long event; - unsigned long unit_mask; - unsigned long kernel; - unsigned long user; - struct perf_event_attr attr; -}; - -static int oprofile_perf_enabled; -static DEFINE_MUTEX(oprofile_perf_mutex); - -static struct op_counter_config *counter_config; -static DEFINE_PER_CPU(struct perf_event **, perf_events); -static int num_counters; - -/* - * Overflow callback for oprofile. - */ -static void op_overflow_handler(struct perf_event *event, - struct perf_sample_data *data, struct pt_regs *regs) -{ - int id; - u32 cpu = smp_processor_id(); - - for (id = 0; id < num_counters; ++id) - if (per_cpu(perf_events, cpu)[id] == event) - break; - - if (id != num_counters) - oprofile_add_sample(regs, id); - else - pr_warn("oprofile: ignoring spurious overflow on cpu %u\n", - cpu); -} - -/* - * Called by oprofile_perf_setup to create perf attributes to mirror the oprofile - * settings in counter_config. Attributes are created as `pinned' events and - * so are permanently scheduled on the PMU. - */ -static void op_perf_setup(void) -{ - int i; - u32 size = sizeof(struct perf_event_attr); - struct perf_event_attr *attr; - - for (i = 0; i < num_counters; ++i) { - attr = &counter_config[i].attr; - memset(attr, 0, size); - attr->type = PERF_TYPE_RAW; - attr->size = size; - attr->config = counter_config[i].event; - attr->sample_period = counter_config[i].count; - attr->pinned = 1; - } -} - -static int op_create_counter(int cpu, int event) -{ - struct perf_event *pevent; - - if (!counter_config[event].enabled || per_cpu(perf_events, cpu)[event]) - return 0; - - pevent = perf_event_create_kernel_counter(&counter_config[event].attr, - cpu, NULL, - op_overflow_handler, NULL); - - if (IS_ERR(pevent)) - return PTR_ERR(pevent); - - if (pevent->state != PERF_EVENT_STATE_ACTIVE) { - perf_event_release_kernel(pevent); - pr_warn("oprofile: failed to enable event %d on CPU %d\n", - event, cpu); - return -EBUSY; - } - - per_cpu(perf_events, cpu)[event] = pevent; - - return 0; -} - -static void op_destroy_counter(int cpu, int event) -{ - struct perf_event *pevent = per_cpu(perf_events, cpu)[event]; - - if (pevent) { - perf_event_release_kernel(pevent); - per_cpu(perf_events, cpu)[event] = NULL; - } -} - -/* - * Called by oprofile_perf_start to create active perf events based on the - * perviously configured attributes. - */ -static int op_perf_start(void) -{ - int cpu, event, ret = 0; - - for_each_online_cpu(cpu) { - for (event = 0; event < num_counters; ++event) { - ret = op_create_counter(cpu, event); - if (ret) - return ret; - } - } - - return ret; -} - -/* - * Called by oprofile_perf_stop at the end of a profiling run. - */ -static void op_perf_stop(void) -{ - int cpu, event; - - for_each_online_cpu(cpu) - for (event = 0; event < num_counters; ++event) - op_destroy_counter(cpu, event); -} - -static int oprofile_perf_create_files(struct dentry *root) -{ - unsigned int i; - - for (i = 0; i < num_counters; i++) { - struct dentry *dir; - char buf[4]; - - snprintf(buf, sizeof buf, "%d", i); - dir = oprofilefs_mkdir(root, buf); - oprofilefs_create_ulong(dir, "enabled", &counter_config[i].enabled); - oprofilefs_create_ulong(dir, "event", &counter_config[i].event); - oprofilefs_create_ulong(dir, "count", &counter_config[i].count); - oprofilefs_create_ulong(dir, "unit_mask", &counter_config[i].unit_mask); - oprofilefs_create_ulong(dir, "kernel", &counter_config[i].kernel); - oprofilefs_create_ulong(dir, "user", &counter_config[i].user); - } - - return 0; -} - -static int oprofile_perf_setup(void) -{ - raw_spin_lock(&oprofilefs_lock); - op_perf_setup(); - raw_spin_unlock(&oprofilefs_lock); - return 0; -} - -static int oprofile_perf_start(void) -{ - int ret = -EBUSY; - - mutex_lock(&oprofile_perf_mutex); - if (!oprofile_perf_enabled) { - ret = 0; - op_perf_start(); - oprofile_perf_enabled = 1; - } - mutex_unlock(&oprofile_perf_mutex); - return ret; -} - -static void oprofile_perf_stop(void) -{ - mutex_lock(&oprofile_perf_mutex); - if (oprofile_perf_enabled) - op_perf_stop(); - oprofile_perf_enabled = 0; - mutex_unlock(&oprofile_perf_mutex); -} - -#ifdef CONFIG_PM - -static int oprofile_perf_suspend(struct platform_device *dev, pm_message_t state) -{ - mutex_lock(&oprofile_perf_mutex); - if (oprofile_perf_enabled) - op_perf_stop(); - mutex_unlock(&oprofile_perf_mutex); - return 0; -} - -static int oprofile_perf_resume(struct platform_device *dev) -{ - mutex_lock(&oprofile_perf_mutex); - if (oprofile_perf_enabled && op_perf_start()) - oprofile_perf_enabled = 0; - mutex_unlock(&oprofile_perf_mutex); - return 0; -} - -static struct platform_driver oprofile_driver = { - .driver = { - .name = "oprofile-perf", - }, - .resume = oprofile_perf_resume, - .suspend = oprofile_perf_suspend, -}; - -static struct platform_device *oprofile_pdev; - -static int __init init_driverfs(void) -{ - int ret; - - ret = platform_driver_register(&oprofile_driver); - if (ret) - return ret; - - oprofile_pdev = platform_device_register_simple( - oprofile_driver.driver.name, 0, NULL, 0); - if (IS_ERR(oprofile_pdev)) { - ret = PTR_ERR(oprofile_pdev); - platform_driver_unregister(&oprofile_driver); - } - - return ret; -} - -static void exit_driverfs(void) -{ - platform_device_unregister(oprofile_pdev); - platform_driver_unregister(&oprofile_driver); -} - -#else - -static inline int init_driverfs(void) { return 0; } -static inline void exit_driverfs(void) { } - -#endif /* CONFIG_PM */ - -void oprofile_perf_exit(void) -{ - int cpu, id; - struct perf_event *event; - - for_each_possible_cpu(cpu) { - for (id = 0; id < num_counters; ++id) { - event = per_cpu(perf_events, cpu)[id]; - if (event) - perf_event_release_kernel(event); - } - - kfree(per_cpu(perf_events, cpu)); - } - - kfree(counter_config); - exit_driverfs(); -} - -int __init oprofile_perf_init(struct oprofile_operations *ops) -{ - int cpu, ret = 0; - - ret = init_driverfs(); - if (ret) - return ret; - - num_counters = perf_num_counters(); - if (num_counters <= 0) { - pr_info("oprofile: no performance counters\n"); - ret = -ENODEV; - goto out; - } - - counter_config = kcalloc(num_counters, - sizeof(struct op_counter_config), GFP_KERNEL); - - if (!counter_config) { - pr_info("oprofile: failed to allocate %d " - "counters\n", num_counters); - ret = -ENOMEM; - num_counters = 0; - goto out; - } - - for_each_possible_cpu(cpu) { - per_cpu(perf_events, cpu) = kcalloc(num_counters, - sizeof(struct perf_event *), GFP_KERNEL); - if (!per_cpu(perf_events, cpu)) { - pr_info("oprofile: failed to allocate %d perf events " - "for cpu %d\n", num_counters, cpu); - ret = -ENOMEM; - goto out; - } - } - - ops->create_files = oprofile_perf_create_files; - ops->setup = oprofile_perf_setup; - ops->start = oprofile_perf_start; - ops->stop = oprofile_perf_stop; - ops->shutdown = oprofile_perf_stop; - ops->cpu_type = op_name_from_perf_id(); - - if (!ops->cpu_type) - ret = -ENODEV; - else - pr_info("oprofile: using %s\n", ops->cpu_type); - -out: - if (ret) - oprofile_perf_exit(); - - return ret; -} diff --git a/drivers/oprofile/oprofile_stats.c b/drivers/oprofile/oprofile_stats.c deleted file mode 100644 index 59659cea4582..000000000000 --- a/drivers/oprofile/oprofile_stats.c +++ /dev/null @@ -1,84 +0,0 @@ -/** - * @file oprofile_stats.c - * - * @remark Copyright 2002 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - */ - -#include -#include -#include -#include - -#include "oprofile_stats.h" -#include "cpu_buffer.h" - -struct oprofile_stat_struct oprofile_stats; - -void oprofile_reset_stats(void) -{ - struct oprofile_cpu_buffer *cpu_buf; - int i; - - for_each_possible_cpu(i) { - cpu_buf = &per_cpu(op_cpu_buffer, i); - cpu_buf->sample_received = 0; - cpu_buf->sample_lost_overflow = 0; - cpu_buf->backtrace_aborted = 0; - cpu_buf->sample_invalid_eip = 0; - } - - atomic_set(&oprofile_stats.sample_lost_no_mm, 0); - atomic_set(&oprofile_stats.sample_lost_no_mapping, 0); - atomic_set(&oprofile_stats.event_lost_overflow, 0); - atomic_set(&oprofile_stats.bt_lost_no_mapping, 0); - atomic_set(&oprofile_stats.multiplex_counter, 0); -} - - -void oprofile_create_stats_files(struct dentry *root) -{ - struct oprofile_cpu_buffer *cpu_buf; - struct dentry *cpudir; - struct dentry *dir; - char buf[10]; - int i; - - dir = oprofilefs_mkdir(root, "stats"); - if (!dir) - return; - - for_each_possible_cpu(i) { - cpu_buf = &per_cpu(op_cpu_buffer, i); - snprintf(buf, 10, "cpu%d", i); - cpudir = oprofilefs_mkdir(dir, buf); - - /* Strictly speaking access to these ulongs is racy, - * but we can't simply lock them, and they are - * informational only. - */ - oprofilefs_create_ro_ulong(cpudir, "sample_received", - &cpu_buf->sample_received); - oprofilefs_create_ro_ulong(cpudir, "sample_lost_overflow", - &cpu_buf->sample_lost_overflow); - oprofilefs_create_ro_ulong(cpudir, "backtrace_aborted", - &cpu_buf->backtrace_aborted); - oprofilefs_create_ro_ulong(cpudir, "sample_invalid_eip", - &cpu_buf->sample_invalid_eip); - } - - oprofilefs_create_ro_atomic(dir, "sample_lost_no_mm", - &oprofile_stats.sample_lost_no_mm); - oprofilefs_create_ro_atomic(dir, "sample_lost_no_mapping", - &oprofile_stats.sample_lost_no_mapping); - oprofilefs_create_ro_atomic(dir, "event_lost_overflow", - &oprofile_stats.event_lost_overflow); - oprofilefs_create_ro_atomic(dir, "bt_lost_no_mapping", - &oprofile_stats.bt_lost_no_mapping); -#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX - oprofilefs_create_ro_atomic(dir, "multiplex_counter", - &oprofile_stats.multiplex_counter); -#endif -} diff --git a/drivers/oprofile/oprofile_stats.h b/drivers/oprofile/oprofile_stats.h deleted file mode 100644 index 1fc622bd1834..000000000000 --- a/drivers/oprofile/oprofile_stats.h +++ /dev/null @@ -1,33 +0,0 @@ -/** - * @file oprofile_stats.h - * - * @remark Copyright 2002 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - */ - -#ifndef OPROFILE_STATS_H -#define OPROFILE_STATS_H - -#include - -struct oprofile_stat_struct { - atomic_t sample_lost_no_mm; - atomic_t sample_lost_no_mapping; - atomic_t bt_lost_no_mapping; - atomic_t event_lost_overflow; - atomic_t multiplex_counter; -}; - -extern struct oprofile_stat_struct oprofile_stats; - -/* reset all stats to zero */ -void oprofile_reset_stats(void); - -struct dentry; - -/* create the stats/ dir */ -void oprofile_create_stats_files(struct dentry *root); - -#endif /* OPROFILE_STATS_H */ diff --git a/drivers/oprofile/oprofilefs.c b/drivers/oprofile/oprofilefs.c deleted file mode 100644 index 0875f2f122b3..000000000000 --- a/drivers/oprofile/oprofilefs.c +++ /dev/null @@ -1,300 +0,0 @@ -/** - * @file oprofilefs.c - * - * @remark Copyright 2002 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - * - * A simple filesystem for configuration and - * access of oprofile. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "oprof.h" - -#define OPROFILEFS_MAGIC 0x6f70726f - -DEFINE_RAW_SPINLOCK(oprofilefs_lock); - -static struct inode *oprofilefs_get_inode(struct super_block *sb, int mode) -{ - struct inode *inode = new_inode(sb); - - if (inode) { - inode->i_ino = get_next_ino(); - inode->i_mode = mode; - inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); - } - return inode; -} - - -static const struct super_operations s_ops = { - .statfs = simple_statfs, - .drop_inode = generic_delete_inode, -}; - - -ssize_t oprofilefs_str_to_user(char const *str, char __user *buf, size_t count, loff_t *offset) -{ - return simple_read_from_buffer(buf, count, offset, str, strlen(str)); -} - - -#define TMPBUFSIZE 50 - -ssize_t oprofilefs_ulong_to_user(unsigned long val, char __user *buf, size_t count, loff_t *offset) -{ - char tmpbuf[TMPBUFSIZE]; - size_t maxlen = snprintf(tmpbuf, TMPBUFSIZE, "%lu\n", val); - if (maxlen > TMPBUFSIZE) - maxlen = TMPBUFSIZE; - return simple_read_from_buffer(buf, count, offset, tmpbuf, maxlen); -} - - -/* - * Note: If oprofilefs_ulong_from_user() returns 0, then *val remains - * unchanged and might be uninitialized. This follows write syscall - * implementation when count is zero: "If count is zero ... [and if] - * no errors are detected, 0 will be returned without causing any - * other effect." (man 2 write) - */ -int oprofilefs_ulong_from_user(unsigned long *val, char const __user *buf, size_t count) -{ - char tmpbuf[TMPBUFSIZE]; - unsigned long flags; - - if (!count) - return 0; - - if (count > TMPBUFSIZE - 1) - return -EINVAL; - - memset(tmpbuf, 0x0, TMPBUFSIZE); - - if (copy_from_user(tmpbuf, buf, count)) - return -EFAULT; - - raw_spin_lock_irqsave(&oprofilefs_lock, flags); - *val = simple_strtoul(tmpbuf, NULL, 0); - raw_spin_unlock_irqrestore(&oprofilefs_lock, flags); - return count; -} - - -static ssize_t ulong_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) -{ - unsigned long *val = file->private_data; - return oprofilefs_ulong_to_user(*val, buf, count, offset); -} - - -static ssize_t ulong_write_file(struct file *file, char const __user *buf, size_t count, loff_t *offset) -{ - unsigned long value; - int retval; - - if (*offset) - return -EINVAL; - - retval = oprofilefs_ulong_from_user(&value, buf, count); - if (retval <= 0) - return retval; - - retval = oprofile_set_ulong(file->private_data, value); - if (retval) - return retval; - - return count; -} - - -static const struct file_operations ulong_fops = { - .read = ulong_read_file, - .write = ulong_write_file, - .open = simple_open, - .llseek = default_llseek, -}; - - -static const struct file_operations ulong_ro_fops = { - .read = ulong_read_file, - .open = simple_open, - .llseek = default_llseek, -}; - - -static int __oprofilefs_create_file(struct dentry *root, char const *name, - const struct file_operations *fops, int perm, void *priv) -{ - struct dentry *dentry; - struct inode *inode; - - if (!root) - return -ENOMEM; - - inode_lock(d_inode(root)); - dentry = d_alloc_name(root, name); - if (!dentry) { - inode_unlock(d_inode(root)); - return -ENOMEM; - } - inode = oprofilefs_get_inode(root->d_sb, S_IFREG | perm); - if (!inode) { - dput(dentry); - inode_unlock(d_inode(root)); - return -ENOMEM; - } - inode->i_fop = fops; - inode->i_private = priv; - d_add(dentry, inode); - inode_unlock(d_inode(root)); - return 0; -} - - -int oprofilefs_create_ulong(struct dentry *root, - char const *name, unsigned long *val) -{ - return __oprofilefs_create_file(root, name, - &ulong_fops, 0644, val); -} - - -int oprofilefs_create_ro_ulong(struct dentry *root, - char const *name, unsigned long *val) -{ - return __oprofilefs_create_file(root, name, - &ulong_ro_fops, 0444, val); -} - - -static ssize_t atomic_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) -{ - atomic_t *val = file->private_data; - return oprofilefs_ulong_to_user(atomic_read(val), buf, count, offset); -} - - -static const struct file_operations atomic_ro_fops = { - .read = atomic_read_file, - .open = simple_open, - .llseek = default_llseek, -}; - - -int oprofilefs_create_ro_atomic(struct dentry *root, - char const *name, atomic_t *val) -{ - return __oprofilefs_create_file(root, name, - &atomic_ro_fops, 0444, val); -} - - -int oprofilefs_create_file(struct dentry *root, - char const *name, const struct file_operations *fops) -{ - return __oprofilefs_create_file(root, name, fops, 0644, NULL); -} - - -int oprofilefs_create_file_perm(struct dentry *root, - char const *name, const struct file_operations *fops, int perm) -{ - return __oprofilefs_create_file(root, name, fops, perm, NULL); -} - - -struct dentry *oprofilefs_mkdir(struct dentry *parent, char const *name) -{ - struct dentry *dentry; - struct inode *inode; - - inode_lock(d_inode(parent)); - dentry = d_alloc_name(parent, name); - if (!dentry) { - inode_unlock(d_inode(parent)); - return NULL; - } - inode = oprofilefs_get_inode(parent->d_sb, S_IFDIR | 0755); - if (!inode) { - dput(dentry); - inode_unlock(d_inode(parent)); - return NULL; - } - inode->i_op = &simple_dir_inode_operations; - inode->i_fop = &simple_dir_operations; - d_add(dentry, inode); - inode_unlock(d_inode(parent)); - return dentry; -} - - -static int oprofilefs_fill_super(struct super_block *sb, struct fs_context *fc) -{ - struct inode *root_inode; - - sb->s_blocksize = PAGE_SIZE; - sb->s_blocksize_bits = PAGE_SHIFT; - sb->s_magic = OPROFILEFS_MAGIC; - sb->s_op = &s_ops; - sb->s_time_gran = 1; - - root_inode = oprofilefs_get_inode(sb, S_IFDIR | 0755); - if (!root_inode) - return -ENOMEM; - root_inode->i_op = &simple_dir_inode_operations; - root_inode->i_fop = &simple_dir_operations; - sb->s_root = d_make_root(root_inode); - if (!sb->s_root) - return -ENOMEM; - - oprofile_create_files(sb->s_root); - - // FIXME: verify kill_litter_super removes our dentries - return 0; -} - -static int oprofilefs_get_tree(struct fs_context *fc) -{ - return get_tree_single(fc, oprofilefs_fill_super); -} - -static const struct fs_context_operations oprofilefs_context_ops = { - .get_tree = oprofilefs_get_tree, -}; - -static int oprofilefs_init_fs_context(struct fs_context *fc) -{ - fc->ops = &oprofilefs_context_ops; - return 0; -} - -static struct file_system_type oprofilefs_type = { - .owner = THIS_MODULE, - .name = "oprofilefs", - .init_fs_context = oprofilefs_init_fs_context, - .kill_sb = kill_litter_super, -}; -MODULE_ALIAS_FS("oprofilefs"); - - -int __init oprofilefs_register(void) -{ - return register_filesystem(&oprofilefs_type); -} - - -void __exit oprofilefs_unregister(void) -{ - unregister_filesystem(&oprofilefs_type); -} diff --git a/drivers/oprofile/timer_int.c b/drivers/oprofile/timer_int.c deleted file mode 100644 index 2498a6cd7c24..000000000000 --- a/drivers/oprofile/timer_int.c +++ /dev/null @@ -1,122 +0,0 @@ -/** - * @file timer_int.c - * - * @remark Copyright 2002 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "oprof.h" - -static DEFINE_PER_CPU(struct hrtimer, oprofile_hrtimer); -static int ctr_running; - -static enum hrtimer_restart oprofile_hrtimer_notify(struct hrtimer *hrtimer) -{ - oprofile_add_sample(get_irq_regs(), 0); - hrtimer_forward_now(hrtimer, ns_to_ktime(TICK_NSEC)); - return HRTIMER_RESTART; -} - -static void __oprofile_hrtimer_start(void *unused) -{ - struct hrtimer *hrtimer = this_cpu_ptr(&oprofile_hrtimer); - - if (!ctr_running) - return; - - hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - hrtimer->function = oprofile_hrtimer_notify; - - hrtimer_start(hrtimer, ns_to_ktime(TICK_NSEC), - HRTIMER_MODE_REL_PINNED); -} - -static int oprofile_hrtimer_start(void) -{ - get_online_cpus(); - ctr_running = 1; - on_each_cpu(__oprofile_hrtimer_start, NULL, 1); - put_online_cpus(); - return 0; -} - -static void __oprofile_hrtimer_stop(int cpu) -{ - struct hrtimer *hrtimer = &per_cpu(oprofile_hrtimer, cpu); - - if (!ctr_running) - return; - - hrtimer_cancel(hrtimer); -} - -static void oprofile_hrtimer_stop(void) -{ - int cpu; - - get_online_cpus(); - for_each_online_cpu(cpu) - __oprofile_hrtimer_stop(cpu); - ctr_running = 0; - put_online_cpus(); -} - -static int oprofile_timer_online(unsigned int cpu) -{ - local_irq_disable(); - __oprofile_hrtimer_start(NULL); - local_irq_enable(); - return 0; -} - -static int oprofile_timer_prep_down(unsigned int cpu) -{ - __oprofile_hrtimer_stop(cpu); - return 0; -} - -static enum cpuhp_state hp_online; - -static int oprofile_hrtimer_setup(void) -{ - int ret; - - ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, - "oprofile/timer:online", - oprofile_timer_online, - oprofile_timer_prep_down); - if (ret < 0) - return ret; - hp_online = ret; - return 0; -} - -static void oprofile_hrtimer_shutdown(void) -{ - cpuhp_remove_state_nocalls(hp_online); -} - -int oprofile_timer_init(struct oprofile_operations *ops) -{ - ops->create_files = NULL; - ops->setup = oprofile_hrtimer_setup; - ops->shutdown = oprofile_hrtimer_shutdown; - ops->start = oprofile_hrtimer_start; - ops->stop = oprofile_hrtimer_stop; - ops->cpu_type = "timer"; - printk(KERN_INFO "oprofile: using timer interrupt.\n"); - return 0; -} diff --git a/include/linux/oprofile.h b/include/linux/oprofile.h deleted file mode 100644 index b2a0f15f11fe..000000000000 --- a/include/linux/oprofile.h +++ /dev/null @@ -1,209 +0,0 @@ -/** - * @file oprofile.h - * - * API for machine-specific interrupts to interface - * to oprofile. - * - * @remark Copyright 2002 OProfile authors - * @remark Read the file COPYING - * - * @author John Levon - */ - -#ifndef OPROFILE_H -#define OPROFILE_H - -#include -#include -#include -#include -#include -#include - -/* Each escaped entry is prefixed by ESCAPE_CODE - * then one of the following codes, then the - * relevant data. - * These #defines live in this file so that arch-specific - * buffer sync'ing code can access them. - */ -#define ESCAPE_CODE ~0UL -#define CTX_SWITCH_CODE 1 -#define CPU_SWITCH_CODE 2 -#define COOKIE_SWITCH_CODE 3 -#define KERNEL_ENTER_SWITCH_CODE 4 -#define KERNEL_EXIT_SWITCH_CODE 5 -#define MODULE_LOADED_CODE 6 -#define CTX_TGID_CODE 7 -#define TRACE_BEGIN_CODE 8 -#define TRACE_END_CODE 9 -#define XEN_ENTER_SWITCH_CODE 10 -#define SPU_PROFILING_CODE 11 -#define SPU_CTX_SWITCH_CODE 12 -#define IBS_FETCH_CODE 13 -#define IBS_OP_CODE 14 - -struct dentry; -struct file_operations; -struct pt_regs; - -/* Operations structure to be filled in */ -struct oprofile_operations { - /* create any necessary configuration files in the oprofile fs. - * Optional. */ - int (*create_files)(struct dentry * root); - /* Do any necessary interrupt setup. Optional. */ - int (*setup)(void); - /* Do any necessary interrupt shutdown. Optional. */ - void (*shutdown)(void); - /* Start delivering interrupts. */ - int (*start)(void); - /* Stop delivering interrupts. */ - void (*stop)(void); - /* Arch-specific buffer sync functions. - * Return value = 0: Success - * Return value = -1: Failure - * Return value = 1: Run generic sync function - */ - int (*sync_start)(void); - int (*sync_stop)(void); - - /* Initiate a stack backtrace. Optional. */ - void (*backtrace)(struct pt_regs * const regs, unsigned int depth); - - /* Multiplex between different events. Optional. */ - int (*switch_events)(void); - /* CPU identification string. */ - char * cpu_type; -}; - -/** - * One-time initialisation. *ops must be set to a filled-in - * operations structure. This is called even in timer interrupt - * mode so an arch can set a backtrace callback. - * - * If an error occurs, the fields should be left untouched. - */ -int oprofile_arch_init(struct oprofile_operations * ops); - -/** - * One-time exit/cleanup for the arch. - */ -void oprofile_arch_exit(void); - -/** - * Add a sample. This may be called from any context. - */ -void oprofile_add_sample(struct pt_regs * const regs, unsigned long event); - -/** - * Add an extended sample. Use this when the PC is not from the regs, and - * we cannot determine if we're in kernel mode from the regs. - * - * This function does perform a backtrace. - * - */ -void oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs, - unsigned long event, int is_kernel); - -/** - * Add an hardware sample. - */ -void oprofile_add_ext_hw_sample(unsigned long pc, struct pt_regs * const regs, - unsigned long event, int is_kernel, - struct task_struct *task); - -/* Use this instead when the PC value is not from the regs. Doesn't - * backtrace. */ -void oprofile_add_pc(unsigned long pc, int is_kernel, unsigned long event); - -/* add a backtrace entry, to be called from the ->backtrace callback */ -void oprofile_add_trace(unsigned long eip); - - -/** - * Create a file of the given name as a child of the given root, with - * the specified file operations. - */ -int oprofilefs_create_file(struct dentry * root, - char const * name, const struct file_operations * fops); - -int oprofilefs_create_file_perm(struct dentry * root, - char const * name, const struct file_operations * fops, int perm); - -/** Create a file for read/write access to an unsigned long. */ -int oprofilefs_create_ulong(struct dentry * root, - char const * name, ulong * val); - -/** Create a file for read-only access to an unsigned long. */ -int oprofilefs_create_ro_ulong(struct dentry * root, - char const * name, ulong * val); - -/** Create a file for read-only access to an atomic_t. */ -int oprofilefs_create_ro_atomic(struct dentry * root, - char const * name, atomic_t * val); - -/** create a directory */ -struct dentry *oprofilefs_mkdir(struct dentry *parent, char const *name); - -/** - * Write the given asciz string to the given user buffer @buf, updating *offset - * appropriately. Returns bytes written or -EFAULT. - */ -ssize_t oprofilefs_str_to_user(char const * str, char __user * buf, size_t count, loff_t * offset); - -/** - * Convert an unsigned long value into ASCII and copy it to the user buffer @buf, - * updating *offset appropriately. Returns bytes written or -EFAULT. - */ -ssize_t oprofilefs_ulong_to_user(unsigned long val, char __user * buf, size_t count, loff_t * offset); - -/** - * Read an ASCII string for a number from a userspace buffer and fill *val on success. - * Returns 0 on success, < 0 on error. - */ -int oprofilefs_ulong_from_user(unsigned long * val, char const __user * buf, size_t count); - -/** lock for read/write safety */ -extern raw_spinlock_t oprofilefs_lock; - -/** - * Add the contents of a circular buffer to the event buffer. - */ -void oprofile_put_buff(unsigned long *buf, unsigned int start, - unsigned int stop, unsigned int max); - -unsigned long oprofile_get_cpu_buffer_size(void); -void oprofile_cpu_buffer_inc_smpl_lost(void); - -/* cpu buffer functions */ - -struct op_sample; - -struct op_entry { - struct ring_buffer_event *event; - struct op_sample *sample; - unsigned long size; - unsigned long *data; -}; - -void oprofile_write_reserve(struct op_entry *entry, - struct pt_regs * const regs, - unsigned long pc, int code, int size); -int oprofile_add_data(struct op_entry *entry, unsigned long val); -int oprofile_add_data64(struct op_entry *entry, u64 val); -int oprofile_write_commit(struct op_entry *entry); - -#ifdef CONFIG_HW_PERF_EVENTS -int __init oprofile_perf_init(struct oprofile_operations *ops); -void oprofile_perf_exit(void); -char *op_name_from_perf_id(void); -#else -static inline int __init oprofile_perf_init(struct oprofile_operations *ops) -{ - pr_info("oprofile: hardware counters not available\n"); - return -ENODEV; -} -static inline void oprofile_perf_exit(void) { } -#endif /* CONFIG_HW_PERF_EVENTS */ - -#endif /* OPROFILE_H */ diff --git a/init/Kconfig b/init/Kconfig index b77c60f8b963..03eb36256405 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -2024,7 +2024,7 @@ config PROFILING bool "Profiling support" help Say Y here to enable the extended profiling support mechanisms used - by profilers such as OProfile. + by profilers. # # Place an empty function call at each tracepoint site. Can be -- cgit v1.2.3 From be65de6b03aa638c46ea51e9d11a92e4914d8103 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 14 Jan 2021 17:05:31 +0530 Subject: fs: Remove dcookies support The dcookies stuff was only used by the kernel's old oprofile code. Now that oprofile's support is removed from the kernel, there is no need for dcookies as well. Remove it. Suggested-by: Christoph Hellwig Suggested-by: Linus Torvalds Signed-off-by: Viresh Kumar Acked-by: Robert Richter Acked-by: William Cohen Acked-by: Al Viro Acked-by: Thomas Gleixner --- fs/Makefile | 1 - fs/dcookies.c | 356 ----------------------------------------------- include/linux/dcookies.h | 69 --------- kernel/sys.c | 1 - 4 files changed, 427 deletions(-) delete mode 100644 fs/dcookies.c delete mode 100644 include/linux/dcookies.h (limited to 'include/linux') diff --git a/fs/Makefile b/fs/Makefile index 999d1a23f036..3215fe205256 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -64,7 +64,6 @@ obj-$(CONFIG_SYSFS) += sysfs/ obj-$(CONFIG_CONFIGFS_FS) += configfs/ obj-y += devpts/ -obj-$(CONFIG_PROFILING) += dcookies.o obj-$(CONFIG_DLM) += dlm/ # Do not add any filesystems before this line diff --git a/fs/dcookies.c b/fs/dcookies.c deleted file mode 100644 index 6eeb61100a09..000000000000 --- a/fs/dcookies.c +++ /dev/null @@ -1,356 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * dcookies.c - * - * Copyright 2002 John Levon - * - * Persistent cookie-path mappings. These are used by - * profilers to convert a per-task EIP value into something - * non-transitory that can be processed at a later date. - * This is done by locking the dentry/vfsmnt pair in the - * kernel until released by the tasks needing the persistent - * objects. The tag is simply an unsigned long that refers - * to the pair and can be looked up from userspace. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* The dcookies are allocated from a kmem_cache and - * hashed onto a small number of lists. None of the - * code here is particularly performance critical - */ -struct dcookie_struct { - struct path path; - struct list_head hash_list; -}; - -static LIST_HEAD(dcookie_users); -static DEFINE_MUTEX(dcookie_mutex); -static struct kmem_cache *dcookie_cache __read_mostly; -static struct list_head *dcookie_hashtable __read_mostly; -static size_t hash_size __read_mostly; - -static inline int is_live(void) -{ - return !(list_empty(&dcookie_users)); -} - - -/* The dentry is locked, its address will do for the cookie */ -static inline unsigned long dcookie_value(struct dcookie_struct * dcs) -{ - return (unsigned long)dcs->path.dentry; -} - - -static size_t dcookie_hash(unsigned long dcookie) -{ - return (dcookie >> L1_CACHE_SHIFT) & (hash_size - 1); -} - - -static struct dcookie_struct * find_dcookie(unsigned long dcookie) -{ - struct dcookie_struct *found = NULL; - struct dcookie_struct * dcs; - struct list_head * pos; - struct list_head * list; - - list = dcookie_hashtable + dcookie_hash(dcookie); - - list_for_each(pos, list) { - dcs = list_entry(pos, struct dcookie_struct, hash_list); - if (dcookie_value(dcs) == dcookie) { - found = dcs; - break; - } - } - - return found; -} - - -static void hash_dcookie(struct dcookie_struct * dcs) -{ - struct list_head * list = dcookie_hashtable + dcookie_hash(dcookie_value(dcs)); - list_add(&dcs->hash_list, list); -} - - -static struct dcookie_struct *alloc_dcookie(const struct path *path) -{ - struct dcookie_struct *dcs = kmem_cache_alloc(dcookie_cache, - GFP_KERNEL); - struct dentry *d; - if (!dcs) - return NULL; - - d = path->dentry; - spin_lock(&d->d_lock); - d->d_flags |= DCACHE_COOKIE; - spin_unlock(&d->d_lock); - - dcs->path = *path; - path_get(path); - hash_dcookie(dcs); - return dcs; -} - - -/* This is the main kernel-side routine that retrieves the cookie - * value for a dentry/vfsmnt pair. - */ -int get_dcookie(const struct path *path, unsigned long *cookie) -{ - int err = 0; - struct dcookie_struct * dcs; - - mutex_lock(&dcookie_mutex); - - if (!is_live()) { - err = -EINVAL; - goto out; - } - - if (path->dentry->d_flags & DCACHE_COOKIE) { - dcs = find_dcookie((unsigned long)path->dentry); - } else { - dcs = alloc_dcookie(path); - if (!dcs) { - err = -ENOMEM; - goto out; - } - } - - *cookie = dcookie_value(dcs); - -out: - mutex_unlock(&dcookie_mutex); - return err; -} - - -/* And here is where the userspace process can look up the cookie value - * to retrieve the path. - */ -static int do_lookup_dcookie(u64 cookie64, char __user *buf, size_t len) -{ - unsigned long cookie = (unsigned long)cookie64; - int err = -EINVAL; - char * kbuf; - char * path; - size_t pathlen; - struct dcookie_struct * dcs; - - /* we could leak path information to users - * without dir read permission without this - */ - if (!capable(CAP_SYS_ADMIN)) - return -EPERM; - - mutex_lock(&dcookie_mutex); - - if (!is_live()) { - err = -EINVAL; - goto out; - } - - if (!(dcs = find_dcookie(cookie))) - goto out; - - err = -ENOMEM; - kbuf = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!kbuf) - goto out; - - /* FIXME: (deleted) ? */ - path = d_path(&dcs->path, kbuf, PAGE_SIZE); - - mutex_unlock(&dcookie_mutex); - - if (IS_ERR(path)) { - err = PTR_ERR(path); - goto out_free; - } - - err = -ERANGE; - - pathlen = kbuf + PAGE_SIZE - path; - if (pathlen <= len) { - err = pathlen; - if (copy_to_user(buf, path, pathlen)) - err = -EFAULT; - } - -out_free: - kfree(kbuf); - return err; -out: - mutex_unlock(&dcookie_mutex); - return err; -} - -SYSCALL_DEFINE3(lookup_dcookie, u64, cookie64, char __user *, buf, size_t, len) -{ - return do_lookup_dcookie(cookie64, buf, len); -} - -#ifdef CONFIG_COMPAT -COMPAT_SYSCALL_DEFINE4(lookup_dcookie, u32, w0, u32, w1, char __user *, buf, compat_size_t, len) -{ -#ifdef __BIG_ENDIAN - return do_lookup_dcookie(((u64)w0 << 32) | w1, buf, len); -#else - return do_lookup_dcookie(((u64)w1 << 32) | w0, buf, len); -#endif -} -#endif - -static int dcookie_init(void) -{ - struct list_head * d; - unsigned int i, hash_bits; - int err = -ENOMEM; - - dcookie_cache = kmem_cache_create("dcookie_cache", - sizeof(struct dcookie_struct), - 0, 0, NULL); - - if (!dcookie_cache) - goto out; - - dcookie_hashtable = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!dcookie_hashtable) - goto out_kmem; - - err = 0; - - /* - * Find the power-of-two list-heads that can fit into the allocation.. - * We don't guarantee that "sizeof(struct list_head)" is necessarily - * a power-of-two. - */ - hash_size = PAGE_SIZE / sizeof(struct list_head); - hash_bits = 0; - do { - hash_bits++; - } while ((hash_size >> hash_bits) != 0); - hash_bits--; - - /* - * Re-calculate the actual number of entries and the mask - * from the number of bits we can fit. - */ - hash_size = 1UL << hash_bits; - - /* And initialize the newly allocated array */ - d = dcookie_hashtable; - i = hash_size; - do { - INIT_LIST_HEAD(d); - d++; - i--; - } while (i); - -out: - return err; -out_kmem: - kmem_cache_destroy(dcookie_cache); - goto out; -} - - -static void free_dcookie(struct dcookie_struct * dcs) -{ - struct dentry *d = dcs->path.dentry; - - spin_lock(&d->d_lock); - d->d_flags &= ~DCACHE_COOKIE; - spin_unlock(&d->d_lock); - - path_put(&dcs->path); - kmem_cache_free(dcookie_cache, dcs); -} - - -static void dcookie_exit(void) -{ - struct list_head * list; - struct list_head * pos; - struct list_head * pos2; - struct dcookie_struct * dcs; - size_t i; - - for (i = 0; i < hash_size; ++i) { - list = dcookie_hashtable + i; - list_for_each_safe(pos, pos2, list) { - dcs = list_entry(pos, struct dcookie_struct, hash_list); - list_del(&dcs->hash_list); - free_dcookie(dcs); - } - } - - kfree(dcookie_hashtable); - kmem_cache_destroy(dcookie_cache); -} - - -struct dcookie_user { - struct list_head next; -}; - -struct dcookie_user * dcookie_register(void) -{ - struct dcookie_user * user; - - mutex_lock(&dcookie_mutex); - - user = kmalloc(sizeof(struct dcookie_user), GFP_KERNEL); - if (!user) - goto out; - - if (!is_live() && dcookie_init()) - goto out_free; - - list_add(&user->next, &dcookie_users); - -out: - mutex_unlock(&dcookie_mutex); - return user; -out_free: - kfree(user); - user = NULL; - goto out; -} - - -void dcookie_unregister(struct dcookie_user * user) -{ - mutex_lock(&dcookie_mutex); - - list_del(&user->next); - kfree(user); - - if (!is_live()) - dcookie_exit(); - - mutex_unlock(&dcookie_mutex); -} - -EXPORT_SYMBOL_GPL(dcookie_register); -EXPORT_SYMBOL_GPL(dcookie_unregister); -EXPORT_SYMBOL_GPL(get_dcookie); diff --git a/include/linux/dcookies.h b/include/linux/dcookies.h deleted file mode 100644 index ddfdac20cad0..000000000000 --- a/include/linux/dcookies.h +++ /dev/null @@ -1,69 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * dcookies.h - * - * Persistent cookie-path mappings - * - * Copyright 2002 John Levon - */ - -#ifndef DCOOKIES_H -#define DCOOKIES_H - - -#ifdef CONFIG_PROFILING - -#include -#include - -struct dcookie_user; -struct path; - -/** - * dcookie_register - register a user of dcookies - * - * Register as a dcookie user. Returns %NULL on failure. - */ -struct dcookie_user * dcookie_register(void); - -/** - * dcookie_unregister - unregister a user of dcookies - * - * Unregister as a dcookie user. This may invalidate - * any dcookie values returned from get_dcookie(). - */ -void dcookie_unregister(struct dcookie_user * user); - -/** - * get_dcookie - acquire a dcookie - * - * Convert the given dentry/vfsmount pair into - * a cookie value. - * - * Returns -EINVAL if no living task has registered as a - * dcookie user. - * - * Returns 0 on success, with *cookie filled in - */ -int get_dcookie(const struct path *path, unsigned long *cookie); - -#else - -static inline struct dcookie_user * dcookie_register(void) -{ - return NULL; -} - -static inline void dcookie_unregister(struct dcookie_user * user) -{ - return; -} - -static inline int get_dcookie(const struct path *path, unsigned long *cookie) -{ - return -ENOSYS; -} - -#endif /* CONFIG_PROFILING */ - -#endif /* DCOOKIES_H */ diff --git a/kernel/sys.c b/kernel/sys.c index 51f00fe20e4d..6928d23c46ea 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include -- cgit v1.2.3 From 1d3f9bb1be8556bce237934ae82b3cdeda3242fd Mon Sep 17 00:00:00 2001 From: dingsenjie Date: Wed, 27 Jan 2021 10:28:01 +0800 Subject: linux/qed: fix spelling typo in qed_chain.h allocted -> allocated Signed-off-by: dingsenjie Link: https://lore.kernel.org/r/20210127022801.8028-1-dingsenjie@163.com Signed-off-by: Jakub Kicinski --- include/linux/qed/qed_chain.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/qed/qed_chain.h b/include/linux/qed/qed_chain.h index 4d58dc8943f0..e339b48de32d 100644 --- a/include/linux/qed/qed_chain.h +++ b/include/linux/qed/qed_chain.h @@ -470,7 +470,7 @@ static inline void *qed_chain_consume(struct qed_chain *p_chain) /** * @brief qed_chain_reset - Resets the chain to its start state * - * @param p_chain pointer to a previously allocted chain + * @param p_chain pointer to a previously allocated chain */ static inline void qed_chain_reset(struct qed_chain *p_chain) { -- cgit v1.2.3 From ae8eba8b5d723a4ca543024b6e51f4d0f4fb6b6b Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 27 Jan 2021 23:53:43 +0000 Subject: tlb: mmu_gather: Remove unused start/end arguments from tlb_finish_mmu() Since commit 7a30df49f63a ("mm: mmu_gather: remove __tlb_reset_range() for force flush"), the 'start' and 'end' arguments to tlb_finish_mmu() are no longer used, since we flush the whole mm in case of a nested invalidation. Remove the unused arguments and update all callers. Signed-off-by: Will Deacon Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Yu Zhao Acked-by: Peter Zijlstra (Intel) Acked-by: Linus Torvalds Link: https://lkml.kernel.org/r/20210127235347.1402-3-will@kernel.org --- arch/ia64/include/asm/tlb.h | 2 +- arch/x86/kernel/ldt.c | 2 +- fs/exec.c | 2 +- include/linux/mm_types.h | 3 +-- mm/hugetlb.c | 2 +- mm/madvise.c | 6 +++--- mm/memory.c | 4 ++-- mm/mmap.c | 4 ++-- mm/mmu_gather.c | 5 +---- mm/oom_kill.c | 4 ++-- 10 files changed, 15 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/arch/ia64/include/asm/tlb.h b/arch/ia64/include/asm/tlb.h index 8d9da6f08a62..7059eb2e867a 100644 --- a/arch/ia64/include/asm/tlb.h +++ b/arch/ia64/include/asm/tlb.h @@ -36,7 +36,7 @@ * tlb_end_vma(tlb, vma); * } * } - * tlb_finish_mmu(tlb, start, end); // finish unmap for address space MM + * tlb_finish_mmu(tlb); // finish unmap for address space MM */ #include #include diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c index b8aee71840ae..0d4e1253c9c9 100644 --- a/arch/x86/kernel/ldt.c +++ b/arch/x86/kernel/ldt.c @@ -400,7 +400,7 @@ static void free_ldt_pgtables(struct mm_struct *mm) tlb_gather_mmu(&tlb, mm, start, end); free_pgd_range(&tlb, start, end, start, end); - tlb_finish_mmu(&tlb, start, end); + tlb_finish_mmu(&tlb); #endif } diff --git a/fs/exec.c b/fs/exec.c index 5d4d52039105..69d89a0c35e9 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -725,7 +725,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) free_pgd_range(&tlb, old_start, old_end, new_end, vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); } - tlb_finish_mmu(&tlb, old_start, old_end); + tlb_finish_mmu(&tlb); /* * Shrink the vma to just the new range. Always succeeds. diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 07d9acb5b19c..1fe6a51298a6 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -590,8 +590,7 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) struct mmu_gather; extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end); -extern void tlb_finish_mmu(struct mmu_gather *tlb, - unsigned long start, unsigned long end); +extern void tlb_finish_mmu(struct mmu_gather *tlb); static inline void init_tlb_flush_pending(struct mm_struct *mm) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 18f6ee317900..33db4fa62c7b 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3985,7 +3985,7 @@ void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, tlb_gather_mmu(&tlb, mm, tlb_start, tlb_end); __unmap_hugepage_range(&tlb, vma, start, end, ref_page); - tlb_finish_mmu(&tlb, tlb_start, tlb_end); + tlb_finish_mmu(&tlb); } /* diff --git a/mm/madvise.c b/mm/madvise.c index 6a660858784b..1b68520ea3f4 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -508,7 +508,7 @@ static long madvise_cold(struct vm_area_struct *vma, lru_add_drain(); tlb_gather_mmu(&tlb, mm, start_addr, end_addr); madvise_cold_page_range(&tlb, vma, start_addr, end_addr); - tlb_finish_mmu(&tlb, start_addr, end_addr); + tlb_finish_mmu(&tlb); return 0; } @@ -560,7 +560,7 @@ static long madvise_pageout(struct vm_area_struct *vma, lru_add_drain(); tlb_gather_mmu(&tlb, mm, start_addr, end_addr); madvise_pageout_page_range(&tlb, vma, start_addr, end_addr); - tlb_finish_mmu(&tlb, start_addr, end_addr); + tlb_finish_mmu(&tlb); return 0; } @@ -732,7 +732,7 @@ static int madvise_free_single_vma(struct vm_area_struct *vma, &madvise_free_walk_ops, &tlb); tlb_end_vma(&tlb, vma); mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb, range.start, range.end); + tlb_finish_mmu(&tlb); return 0; } diff --git a/mm/memory.c b/mm/memory.c index feff48e1465a..7bd3f122bd10 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1540,7 +1540,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next) unmap_single_vma(&tlb, vma, start, range.end, NULL); mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb, start, range.end); + tlb_finish_mmu(&tlb); } /** @@ -1566,7 +1566,7 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addr mmu_notifier_invalidate_range_start(&range); unmap_single_vma(&tlb, vma, address, range.end, details); mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb, address, range.end); + tlb_finish_mmu(&tlb); } /** diff --git a/mm/mmap.c b/mm/mmap.c index dc7206032387..7a9f493a4b83 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2676,7 +2676,7 @@ static void unmap_region(struct mm_struct *mm, unmap_vmas(&tlb, vma, start, end); free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, next ? next->vm_start : USER_PGTABLES_CEILING); - tlb_finish_mmu(&tlb, start, end); + tlb_finish_mmu(&tlb); } /* @@ -3219,7 +3219,7 @@ void exit_mmap(struct mm_struct *mm) /* Use -1 here to ensure all VMAs in the mm are unmapped */ unmap_vmas(&tlb, vma, 0, -1); free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING); - tlb_finish_mmu(&tlb, 0, -1); + tlb_finish_mmu(&tlb); /* * Walk the list again, actually closing and freeing it, diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 03c33c93a582..b0be5a7aa08f 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -290,14 +290,11 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, /** * tlb_finish_mmu - finish an mmu_gather structure * @tlb: the mmu_gather structure to finish - * @start: start of the region that will be removed from the page-table - * @end: end of the region that will be removed from the page-table * * Called at the end of the shootdown operation to free up any resources that * were required. */ -void tlb_finish_mmu(struct mmu_gather *tlb, - unsigned long start, unsigned long end) +void tlb_finish_mmu(struct mmu_gather *tlb) { /* * If there are parallel threads are doing PTE changes on same range diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 04b19b7b5435..757e557211fb 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -548,13 +548,13 @@ bool __oom_reap_task_mm(struct mm_struct *mm) vma->vm_end); tlb_gather_mmu(&tlb, mm, range.start, range.end); if (mmu_notifier_invalidate_range_start_nonblock(&range)) { - tlb_finish_mmu(&tlb, range.start, range.end); + tlb_finish_mmu(&tlb); ret = false; continue; } unmap_page_range(&tlb, vma, range.start, range.end, NULL); mmu_notifier_invalidate_range_end(&range); - tlb_finish_mmu(&tlb, range.start, range.end); + tlb_finish_mmu(&tlb); } } -- cgit v1.2.3 From d8b450530b90f8845ab962af18b8a10ed77fc889 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 27 Jan 2021 23:53:44 +0000 Subject: tlb: mmu_gather: Introduce tlb_gather_mmu_fullmm() Passing the range '0, -1' to tlb_gather_mmu() sets the 'fullmm' flag, which indicates that the mm_struct being operated on is going away. In this case, some architectures (such as arm64) can elide TLB invalidation by ensuring that the TLB tag (ASID) associated with this mm is not immediately reclaimed. Although this behaviour is documented in asm-generic/tlb.h, it's subtle and easily missed. Introduce tlb_gather_mmu_fullmm() to make it clearer that this is for the entire mm and WARN() if tlb_gather_mmu() is called with the 'fullmm' address range. Signed-off-by: Will Deacon Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Yu Zhao Acked-by: Peter Zijlstra (Intel) Acked-by: Linus Torvalds Link: https://lkml.kernel.org/r/20210127235347.1402-4-will@kernel.org --- include/asm-generic/tlb.h | 6 ++++-- include/linux/mm_types.h | 1 + mm/mmap.c | 2 +- mm/mmu_gather.c | 16 ++++++++++++++-- 4 files changed, 20 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 6661ee1cff47..2c68a545ffa7 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -46,7 +46,9 @@ * * The mmu_gather API consists of: * - * - tlb_gather_mmu() / tlb_finish_mmu(); start and finish a mmu_gather + * - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_finish_mmu() + * + * start and finish a mmu_gather * * Finish in particular will issue a (final) TLB invalidate and free * all (remaining) queued pages. @@ -91,7 +93,7 @@ * * - mmu_gather::fullmm * - * A flag set by tlb_gather_mmu() to indicate we're going to free + * A flag set by tlb_gather_mmu_fullmm() to indicate we're going to free * the entire mm; this allows a number of optimizations. * * - We can ignore tlb_{start,end}_vma(); because we don't diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 1fe6a51298a6..e49868bc12a7 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -590,6 +590,7 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) struct mmu_gather; extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end); +extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_finish_mmu(struct mmu_gather *tlb); static inline void init_tlb_flush_pending(struct mm_struct *mm) diff --git a/mm/mmap.c b/mm/mmap.c index 7a9f493a4b83..4eac7c63edbe 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -3214,7 +3214,7 @@ void exit_mmap(struct mm_struct *mm) lru_add_drain(); flush_cache_mm(mm); - tlb_gather_mmu(&tlb, mm, 0, -1); + tlb_gather_mmu_fullmm(&tlb, mm); /* update_hiwater_rss(mm) here? but nobody should be looking */ /* Use -1 here to ensure all VMAs in the mm are unmapped */ unmap_vmas(&tlb, vma, 0, -1); diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index b0be5a7aa08f..5f5e45d9eb50 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -261,8 +261,8 @@ void tlb_flush_mmu(struct mmu_gather *tlb) * respectively when @mm is without users and we're going to destroy * the full address space (exit/execve). */ -void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, - unsigned long start, unsigned long end) +static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, + unsigned long start, unsigned long end) { tlb->mm = mm; @@ -287,6 +287,18 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, inc_tlb_flush_pending(tlb->mm); } +void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + WARN_ON(!(start | (end + 1))); /* Use _fullmm() instead */ + __tlb_gather_mmu(tlb, mm, start, end); +} + +void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm) +{ + __tlb_gather_mmu(tlb, mm, 0, -1); +} + /** * tlb_finish_mmu - finish an mmu_gather structure * @tlb: the mmu_gather structure to finish -- cgit v1.2.3 From a72afd873089c697053e9daa85ff343b3140d2e7 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 27 Jan 2021 23:53:45 +0000 Subject: tlb: mmu_gather: Remove start/end arguments from tlb_gather_mmu() The 'start' and 'end' arguments to tlb_gather_mmu() are no longer needed now that there is a separate function for 'fullmm' flushing. Remove the unused arguments and update all callers. Suggested-by: Linus Torvalds Signed-off-by: Will Deacon Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Yu Zhao Acked-by: Peter Zijlstra (Intel) Acked-by: Linus Torvalds Link: https://lore.kernel.org/r/CAHk-=wjQWa14_4UpfDf=fiineNP+RH74kZeDMo_f1D35xNzq9w@mail.gmail.com --- arch/ia64/include/asm/tlb.h | 2 +- arch/x86/kernel/ldt.c | 2 +- fs/exec.c | 2 +- include/linux/mm_types.h | 3 +-- mm/hugetlb.c | 16 +--------------- mm/madvise.c | 6 +++--- mm/memory.c | 4 ++-- mm/mmap.c | 2 +- mm/mmu_gather.c | 22 ++++++++-------------- mm/oom_kill.c | 2 +- 10 files changed, 20 insertions(+), 41 deletions(-) (limited to 'include/linux') diff --git a/arch/ia64/include/asm/tlb.h b/arch/ia64/include/asm/tlb.h index 7059eb2e867a..a15fe0809aae 100644 --- a/arch/ia64/include/asm/tlb.h +++ b/arch/ia64/include/asm/tlb.h @@ -23,7 +23,7 @@ * unmapping a portion of the virtual address space, these hooks are called according to * the following template: * - * tlb <- tlb_gather_mmu(mm, start, end); // start unmap for address space MM + * tlb <- tlb_gather_mmu(mm); // start unmap for address space MM * { * for each vma that needs a shootdown do { * tlb_start_vma(tlb, vma); diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c index 0d4e1253c9c9..7ad9834e0d95 100644 --- a/arch/x86/kernel/ldt.c +++ b/arch/x86/kernel/ldt.c @@ -398,7 +398,7 @@ static void free_ldt_pgtables(struct mm_struct *mm) if (!boot_cpu_has(X86_FEATURE_PTI)) return; - tlb_gather_mmu(&tlb, mm, start, end); + tlb_gather_mmu(&tlb, mm); free_pgd_range(&tlb, start, end, start, end); tlb_finish_mmu(&tlb); #endif diff --git a/fs/exec.c b/fs/exec.c index 69d89a0c35e9..5a853f03c233 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -708,7 +708,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) return -ENOMEM; lru_add_drain(); - tlb_gather_mmu(&tlb, mm, old_start, old_end); + tlb_gather_mmu(&tlb, mm); if (new_end > old_start) { /* * when the old and new regions overlap clear from new_end. diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index e49868bc12a7..0974ad501a47 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -588,8 +588,7 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) } struct mmu_gather; -extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, - unsigned long start, unsigned long end); +extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_finish_mmu(struct mmu_gather *tlb); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 33db4fa62c7b..89635f407232 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3967,23 +3967,9 @@ void __unmap_hugepage_range_final(struct mmu_gather *tlb, void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, struct page *ref_page) { - struct mm_struct *mm; struct mmu_gather tlb; - unsigned long tlb_start = start; - unsigned long tlb_end = end; - /* - * If shared PMDs were possibly used within this vma range, adjust - * start/end for worst case tlb flushing. - * Note that we can not be sure if PMDs are shared until we try to - * unmap pages. However, we want to make sure TLB flushing covers - * the largest possible range. - */ - adjust_range_if_pmd_sharing_possible(vma, &tlb_start, &tlb_end); - - mm = vma->vm_mm; - - tlb_gather_mmu(&tlb, mm, tlb_start, tlb_end); + tlb_gather_mmu(&tlb, vma->vm_mm); __unmap_hugepage_range(&tlb, vma, start, end, ref_page); tlb_finish_mmu(&tlb); } diff --git a/mm/madvise.c b/mm/madvise.c index 1b68520ea3f4..0938fd3ad228 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -506,7 +506,7 @@ static long madvise_cold(struct vm_area_struct *vma, return -EINVAL; lru_add_drain(); - tlb_gather_mmu(&tlb, mm, start_addr, end_addr); + tlb_gather_mmu(&tlb, mm); madvise_cold_page_range(&tlb, vma, start_addr, end_addr); tlb_finish_mmu(&tlb); @@ -558,7 +558,7 @@ static long madvise_pageout(struct vm_area_struct *vma, return 0; lru_add_drain(); - tlb_gather_mmu(&tlb, mm, start_addr, end_addr); + tlb_gather_mmu(&tlb, mm); madvise_pageout_page_range(&tlb, vma, start_addr, end_addr); tlb_finish_mmu(&tlb); @@ -723,7 +723,7 @@ static int madvise_free_single_vma(struct vm_area_struct *vma, range.start, range.end); lru_add_drain(); - tlb_gather_mmu(&tlb, mm, range.start, range.end); + tlb_gather_mmu(&tlb, mm); update_hiwater_rss(mm); mmu_notifier_invalidate_range_start(&range); diff --git a/mm/memory.c b/mm/memory.c index 7bd3f122bd10..9e8576a83147 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1534,7 +1534,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, lru_add_drain(); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, start, start + size); - tlb_gather_mmu(&tlb, vma->vm_mm, start, range.end); + tlb_gather_mmu(&tlb, vma->vm_mm); update_hiwater_rss(vma->vm_mm); mmu_notifier_invalidate_range_start(&range); for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next) @@ -1561,7 +1561,7 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addr lru_add_drain(); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, address, address + size); - tlb_gather_mmu(&tlb, vma->vm_mm, address, range.end); + tlb_gather_mmu(&tlb, vma->vm_mm); update_hiwater_rss(vma->vm_mm); mmu_notifier_invalidate_range_start(&range); unmap_single_vma(&tlb, vma, address, range.end, details); diff --git a/mm/mmap.c b/mm/mmap.c index 4eac7c63edbe..90673febce6a 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2671,7 +2671,7 @@ static void unmap_region(struct mm_struct *mm, struct mmu_gather tlb; lru_add_drain(); - tlb_gather_mmu(&tlb, mm, start, end); + tlb_gather_mmu(&tlb, mm); update_hiwater_rss(mm); unmap_vmas(&tlb, vma, start, end); free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 5f5e45d9eb50..0dc7149b0c61 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -253,21 +253,17 @@ void tlb_flush_mmu(struct mmu_gather *tlb) * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down * @tlb: the mmu_gather structure to initialize * @mm: the mm_struct of the target address space - * @start: start of the region that will be removed from the page-table - * @end: end of the region that will be removed from the page-table + * @fullmm: @mm is without users and we're going to destroy the full address + * space (exit/execve) * * Called to initialize an (on-stack) mmu_gather structure for page-table - * tear-down from @mm. The @start and @end are set to 0 and -1 - * respectively when @mm is without users and we're going to destroy - * the full address space (exit/execve). + * tear-down from @mm. */ static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, - unsigned long start, unsigned long end) + bool fullmm) { tlb->mm = mm; - - /* Is it from 0 to ~0? */ - tlb->fullmm = !(start | (end+1)); + tlb->fullmm = fullmm; #ifndef CONFIG_MMU_GATHER_NO_GATHER tlb->need_flush_all = 0; @@ -287,16 +283,14 @@ static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, inc_tlb_flush_pending(tlb->mm); } -void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, - unsigned long start, unsigned long end) +void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm) { - WARN_ON(!(start | (end + 1))); /* Use _fullmm() instead */ - __tlb_gather_mmu(tlb, mm, start, end); + __tlb_gather_mmu(tlb, mm, false); } void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm) { - __tlb_gather_mmu(tlb, mm, 0, -1); + __tlb_gather_mmu(tlb, mm, true); } /** diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 757e557211fb..c9a33ffe38b7 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -546,7 +546,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, vma->vm_start, vma->vm_end); - tlb_gather_mmu(&tlb, mm, range.start, range.end); + tlb_gather_mmu(&tlb, mm); if (mmu_notifier_invalidate_range_start_nonblock(&range)) { tlb_finish_mmu(&tlb); ret = false; -- cgit v1.2.3 From 97c753e62e6c31a404183898d950d8c08d752dbd Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Thu, 28 Jan 2021 00:37:51 +0900 Subject: tracing/kprobe: Fix to support kretprobe events on unloaded modules Fix kprobe_on_func_entry() returns error code instead of false so that register_kretprobe() can return an appropriate error code. append_trace_kprobe() expects the kprobe registration returns -ENOENT when the target symbol is not found, and it checks whether the target module is unloaded or not. If the target module doesn't exist, it defers to probe the target symbol until the module is loaded. However, since register_kretprobe() returns -EINVAL instead of -ENOENT in that case, it always fail on putting the kretprobe event on unloaded modules. e.g. Kprobe event: /sys/kernel/debug/tracing # echo p xfs:xfs_end_io >> kprobe_events [ 16.515574] trace_kprobe: This probe might be able to register after target module is loaded. Continue. Kretprobe event: (p -> r) /sys/kernel/debug/tracing # echo r xfs:xfs_end_io >> kprobe_events sh: write error: Invalid argument /sys/kernel/debug/tracing # cat error_log [ 41.122514] trace_kprobe: error: Failed to register probe event Command: r xfs:xfs_end_io ^ To fix this bug, change kprobe_on_func_entry() to detect symbol lookup failure and return -ENOENT in that case. Otherwise it returns -EINVAL or 0 (succeeded, given address is on the entry). Link: https://lkml.kernel.org/r/161176187132.1067016.8118042342894378981.stgit@devnote2 Cc: stable@vger.kernel.org Fixes: 59158ec4aef7 ("tracing/kprobes: Check the probe on unloaded module correctly") Reported-by: Jianlin Lv Signed-off-by: Masami Hiramatsu Signed-off-by: Steven Rostedt (VMware) --- include/linux/kprobes.h | 2 +- kernel/kprobes.c | 34 +++++++++++++++++++++++++--------- kernel/trace/trace_kprobe.c | 10 ++++++---- 3 files changed, 32 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h index b3a36b0cfc81..1883a4a9f16a 100644 --- a/include/linux/kprobes.h +++ b/include/linux/kprobes.h @@ -266,7 +266,7 @@ extern void kprobes_inc_nmissed_count(struct kprobe *p); extern bool arch_within_kprobe_blacklist(unsigned long addr); extern int arch_populate_kprobe_blacklist(void); extern bool arch_kprobe_on_func_entry(unsigned long offset); -extern bool kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset); +extern int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset); extern bool within_kprobe_blacklist(unsigned long addr); extern int kprobe_add_ksym_blacklist(unsigned long entry); diff --git a/kernel/kprobes.c b/kernel/kprobes.c index f7fb5d135930..1a5bc321e0a5 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -1954,29 +1954,45 @@ bool __weak arch_kprobe_on_func_entry(unsigned long offset) return !offset; } -bool kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset) +/** + * kprobe_on_func_entry() -- check whether given address is function entry + * @addr: Target address + * @sym: Target symbol name + * @offset: The offset from the symbol or the address + * + * This checks whether the given @addr+@offset or @sym+@offset is on the + * function entry address or not. + * This returns 0 if it is the function entry, or -EINVAL if it is not. + * And also it returns -ENOENT if it fails the symbol or address lookup. + * Caller must pass @addr or @sym (either one must be NULL), or this + * returns -EINVAL. + */ +int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset) { kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset); if (IS_ERR(kp_addr)) - return false; + return PTR_ERR(kp_addr); - if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset) || - !arch_kprobe_on_func_entry(offset)) - return false; + if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset)) + return -ENOENT; - return true; + if (!arch_kprobe_on_func_entry(offset)) + return -EINVAL; + + return 0; } int register_kretprobe(struct kretprobe *rp) { - int ret = 0; + int ret; struct kretprobe_instance *inst; int i; void *addr; - if (!kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset)) - return -EINVAL; + ret = kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset); + if (ret) + return ret; if (kretprobe_blacklist_size) { addr = kprobe_addr(&rp->kp); diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index e6fba1798771..56c7fbff7bd7 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -221,9 +221,9 @@ bool trace_kprobe_on_func_entry(struct trace_event_call *call) { struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); - return tk ? kprobe_on_func_entry(tk->rp.kp.addr, + return tk ? (kprobe_on_func_entry(tk->rp.kp.addr, tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name, - tk->rp.kp.addr ? 0 : tk->rp.kp.offset) : false; + tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false; } bool trace_kprobe_error_injectable(struct trace_event_call *call) @@ -828,9 +828,11 @@ static int trace_kprobe_create(int argc, const char *argv[]) } if (is_return) flags |= TPARG_FL_RETURN; - if (kprobe_on_func_entry(NULL, symbol, offset)) + ret = kprobe_on_func_entry(NULL, symbol, offset); + if (ret == 0) flags |= TPARG_FL_FENTRY; - if (offset && is_return && !(flags & TPARG_FL_FENTRY)) { + /* Defer the ENOENT case until register kprobe */ + if (ret == -EINVAL && is_return) { trace_probe_log_err(0, BAD_RETPROBE); goto parse_error; } -- cgit v1.2.3 From 4c457e8cb75eda91906a4f89fc39bde3f9a43922 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 23 Jan 2021 12:27:59 +0000 Subject: genirq/msi: Activate Multi-MSI early when MSI_FLAG_ACTIVATE_EARLY is set When MSI_FLAG_ACTIVATE_EARLY is set (which is the case for PCI), __msi_domain_alloc_irqs() performs the activation of the interrupt (which in the case of PCI results in the endpoint being programmed) as soon as the interrupt is allocated. But it appears that this is only done for the first vector, introducing an inconsistent behaviour for PCI Multi-MSI. Fix it by iterating over the number of vectors allocated to each MSI descriptor. This is easily achieved by introducing a new "for_each_msi_vector" iterator, together with a tiny bit of refactoring. Fixes: f3b0946d629c ("genirq/msi: Make sure PCI MSIs are activated early") Reported-by: Shameer Kolothum Signed-off-by: Marc Zyngier Signed-off-by: Thomas Gleixner Tested-by: Shameer Kolothum Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20210123122759.1781359-1-maz@kernel.org --- include/linux/msi.h | 6 ++++++ kernel/irq/msi.c | 44 ++++++++++++++++++++------------------------ 2 files changed, 26 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/msi.h b/include/linux/msi.h index 360a0a7e7341..aef35fd1cf11 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -178,6 +178,12 @@ struct msi_desc { list_for_each_entry((desc), dev_to_msi_list((dev)), list) #define for_each_msi_entry_safe(desc, tmp, dev) \ list_for_each_entry_safe((desc), (tmp), dev_to_msi_list((dev)), list) +#define for_each_msi_vector(desc, __irq, dev) \ + for_each_msi_entry((desc), (dev)) \ + if ((desc)->irq) \ + for (__irq = (desc)->irq; \ + __irq < ((desc)->irq + (desc)->nvec_used); \ + __irq++) #ifdef CONFIG_IRQ_MSI_IOMMU static inline const void *msi_desc_get_iommu_cookie(struct msi_desc *desc) diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c index dc0e2d7fbdfd..b338d622f26e 100644 --- a/kernel/irq/msi.c +++ b/kernel/irq/msi.c @@ -436,22 +436,22 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, can_reserve = msi_check_reservation_mode(domain, info, dev); - for_each_msi_entry(desc, dev) { - virq = desc->irq; - if (desc->nvec_used == 1) - dev_dbg(dev, "irq %d for MSI\n", virq); - else + /* + * This flag is set by the PCI layer as we need to activate + * the MSI entries before the PCI layer enables MSI in the + * card. Otherwise the card latches a random msi message. + */ + if (!(info->flags & MSI_FLAG_ACTIVATE_EARLY)) + goto skip_activate; + + for_each_msi_vector(desc, i, dev) { + if (desc->irq == i) { + virq = desc->irq; dev_dbg(dev, "irq [%d-%d] for MSI\n", virq, virq + desc->nvec_used - 1); - /* - * This flag is set by the PCI layer as we need to activate - * the MSI entries before the PCI layer enables MSI in the - * card. Otherwise the card latches a random msi message. - */ - if (!(info->flags & MSI_FLAG_ACTIVATE_EARLY)) - continue; + } - irq_data = irq_domain_get_irq_data(domain, desc->irq); + irq_data = irq_domain_get_irq_data(domain, i); if (!can_reserve) { irqd_clr_can_reserve(irq_data); if (domain->flags & IRQ_DOMAIN_MSI_NOMASK_QUIRK) @@ -462,28 +462,24 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, goto cleanup; } +skip_activate: /* * If these interrupts use reservation mode, clear the activated bit * so request_irq() will assign the final vector. */ if (can_reserve) { - for_each_msi_entry(desc, dev) { - irq_data = irq_domain_get_irq_data(domain, desc->irq); + for_each_msi_vector(desc, i, dev) { + irq_data = irq_domain_get_irq_data(domain, i); irqd_clr_activated(irq_data); } } return 0; cleanup: - for_each_msi_entry(desc, dev) { - struct irq_data *irqd; - - if (desc->irq == virq) - break; - - irqd = irq_domain_get_irq_data(domain, desc->irq); - if (irqd_is_activated(irqd)) - irq_domain_deactivate_irq(irqd); + for_each_msi_vector(desc, i, dev) { + irq_data = irq_domain_get_irq_data(domain, i); + if (irqd_is_activated(irq_data)) + irq_domain_deactivate_irq(irq_data); } msi_domain_free_irqs(domain, dev); return ret; -- cgit v1.2.3 From 9c7caf28068421c9e0d1faea437e35e6b8983ac6 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Fri, 29 Jan 2021 02:59:59 +0200 Subject: net: dsa: tag_8021q: add helpers to deduce whether a VLAN ID is RX or TX VLAN The sja1105 implementation can be blind about this, but the felix driver doesn't do exactly what it's being told, so it needs to know whether it is a TX or an RX VLAN, so it can install the appropriate type of TCAM rule. Signed-off-by: Vladimir Oltean Reviewed-by: Florian Fainelli Signed-off-by: Jakub Kicinski --- include/linux/dsa/8021q.h | 14 ++++++++++++++ net/dsa/tag_8021q.c | 15 +++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/dsa/8021q.h b/include/linux/dsa/8021q.h index 88cd72dfa4e0..b12b05f1c8b4 100644 --- a/include/linux/dsa/8021q.h +++ b/include/linux/dsa/8021q.h @@ -64,6 +64,10 @@ int dsa_8021q_rx_source_port(u16 vid); u16 dsa_8021q_rx_subvlan(u16 vid); +bool vid_is_dsa_8021q_rxvlan(u16 vid); + +bool vid_is_dsa_8021q_txvlan(u16 vid); + bool vid_is_dsa_8021q(u16 vid); #else @@ -123,6 +127,16 @@ u16 dsa_8021q_rx_subvlan(u16 vid) return 0; } +bool vid_is_dsa_8021q_rxvlan(u16 vid) +{ + return false; +} + +bool vid_is_dsa_8021q_txvlan(u16 vid) +{ + return false; +} + bool vid_is_dsa_8021q(u16 vid) { return false; diff --git a/net/dsa/tag_8021q.c b/net/dsa/tag_8021q.c index 8e3e8a5b8559..008c1ec6e20c 100644 --- a/net/dsa/tag_8021q.c +++ b/net/dsa/tag_8021q.c @@ -133,10 +133,21 @@ u16 dsa_8021q_rx_subvlan(u16 vid) } EXPORT_SYMBOL_GPL(dsa_8021q_rx_subvlan); +bool vid_is_dsa_8021q_rxvlan(u16 vid) +{ + return (vid & DSA_8021Q_DIR_MASK) == DSA_8021Q_DIR_RX; +} +EXPORT_SYMBOL_GPL(vid_is_dsa_8021q_rxvlan); + +bool vid_is_dsa_8021q_txvlan(u16 vid) +{ + return (vid & DSA_8021Q_DIR_MASK) == DSA_8021Q_DIR_TX; +} +EXPORT_SYMBOL_GPL(vid_is_dsa_8021q_txvlan); + bool vid_is_dsa_8021q(u16 vid) { - return ((vid & DSA_8021Q_DIR_MASK) == DSA_8021Q_DIR_RX || - (vid & DSA_8021Q_DIR_MASK) == DSA_8021Q_DIR_TX); + return vid_is_dsa_8021q_rxvlan(vid) || vid_is_dsa_8021q_txvlan(vid); } EXPORT_SYMBOL_GPL(vid_is_dsa_8021q); -- cgit v1.2.3 From 3cc55f4434b421d37300aa9a167ace7d60b45ccf Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Fri, 29 Jan 2021 14:26:29 -0500 Subject: nfs: use change attribute for NFS re-exports When exporting NFS, we may as well use the real change attribute returned by the original server instead of faking up a change attribute from the ctime. Note we can't do that by setting I_VERSION--that would also turn on the logic in iversion.h which treats the lower bit specially, and that doesn't make sense for NFS. So instead we define a new export operation for filesystems like NFS that want to manage the change attribute themselves. Signed-off-by: J. Bruce Fields Reviewed-by: Christoph Hellwig Signed-off-by: Chuck Lever --- fs/nfs/export.c | 18 ++++++++++++++++++ fs/nfsd/nfsfh.h | 5 ++++- include/linux/exportfs.h | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/fs/nfs/export.c b/fs/nfs/export.c index 7412bb164fa7..f2b34cfe286c 100644 --- a/fs/nfs/export.c +++ b/fs/nfs/export.c @@ -167,10 +167,28 @@ out: return parent; } +static u64 nfs_fetch_iversion(struct inode *inode) +{ + struct nfs_server *server = NFS_SERVER(inode); + + /* Is this the right call?: */ + nfs_revalidate_inode(server, inode); + /* + * Also, note we're ignoring any returned error. That seems to be + * the practice for cache consistency information elsewhere in + * the server, but I'm not sure why. + */ + if (server->nfs_client->rpc_ops->version >= 4) + return inode_peek_iversion_raw(inode); + else + return time_to_chattr(&inode->i_ctime); +} + const struct export_operations nfs_export_ops = { .encode_fh = nfs_encode_fh, .fh_to_dentry = nfs_fh_to_dentry, .get_parent = nfs_get_parent, + .fetch_iversion = nfs_fetch_iversion, .flags = EXPORT_OP_NOWCC|EXPORT_OP_NOSUBTREECHK| EXPORT_OP_CLOSE_BEFORE_UNLINK|EXPORT_OP_REMOTE_FS| EXPORT_OP_NOATOMIC_ATTR, diff --git a/fs/nfsd/nfsfh.h b/fs/nfsd/nfsfh.h index cb20c2cd3469..f58933519f38 100644 --- a/fs/nfsd/nfsfh.h +++ b/fs/nfsd/nfsfh.h @@ -12,6 +12,7 @@ #include #include #include +#include static inline __u32 ino_t_to_u32(ino_t ino) { @@ -264,7 +265,9 @@ fh_clear_wcc(struct svc_fh *fhp) static inline u64 nfsd4_change_attribute(struct kstat *stat, struct inode *inode) { - if (IS_I_VERSION(inode)) { + if (inode->i_sb->s_export_op->fetch_iversion) + return inode->i_sb->s_export_op->fetch_iversion(inode); + else if (IS_I_VERSION(inode)) { u64 chattr; chattr = stat->ctime.tv_sec; diff --git a/include/linux/exportfs.h b/include/linux/exportfs.h index 9f4d4bcbf251..fe848901fcc3 100644 --- a/include/linux/exportfs.h +++ b/include/linux/exportfs.h @@ -213,6 +213,7 @@ struct export_operations { bool write, u32 *device_generation); int (*commit_blocks)(struct inode *inode, struct iomap *iomaps, int nr_iomaps, struct iattr *iattr); + u64 (*fetch_iversion)(struct inode *); #define EXPORT_OP_NOWCC (0x1) /* don't collect v3 wcc data */ #define EXPORT_OP_NOSUBTREECHK (0x2) /* no subtree checking */ #define EXPORT_OP_CLOSE_BEFORE_UNLINK (0x4) /* close files before unlink */ -- cgit v1.2.3 From 207f13b419a60c56fb75baeb3d668de080514354 Mon Sep 17 00:00:00 2001 From: Fabien Parent Date: Wed, 9 Dec 2020 14:32:37 +0100 Subject: soc: mediatek: pm-domains: Add support for mt8167 Add the needed board data to support mt8167 SoC. Signed-off-by: Fabien Parent Reviewed-by: Enric Balletbo i Serra Link: https://lore.kernel.org/r/20201209133238.384030-2-fparent@baylibre.com Signed-off-by: Matthias Brugger --- drivers/soc/mediatek/mt8167-pm-domains.h | 86 ++++++++++++++++++++++++++++++++ drivers/soc/mediatek/mtk-pm-domains.c | 5 ++ drivers/soc/mediatek/mtk-pm-domains.h | 1 + include/linux/soc/mediatek/infracfg.h | 8 +++ 4 files changed, 100 insertions(+) create mode 100644 drivers/soc/mediatek/mt8167-pm-domains.h (limited to 'include/linux') diff --git a/drivers/soc/mediatek/mt8167-pm-domains.h b/drivers/soc/mediatek/mt8167-pm-domains.h new file mode 100644 index 000000000000..ad0b8dfa0527 --- /dev/null +++ b/drivers/soc/mediatek/mt8167-pm-domains.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __SOC_MEDIATEK_MT8167_PM_DOMAINS_H +#define __SOC_MEDIATEK_MT8167_PM_DOMAINS_H + +#include "mtk-pm-domains.h" +#include + +#define MT8167_PWR_STATUS_MFG_2D BIT(24) +#define MT8167_PWR_STATUS_MFG_ASYNC BIT(25) + +/* + * MT8167 power domain support + */ + +static const struct scpsys_domain_data scpsys_domain_data_mt8167[] = { + [MT8167_POWER_DOMAIN_MM] = { + .sta_mask = PWR_STATUS_DISP, + .ctl_offs = SPM_DIS_PWR_CON, + .sram_pdn_bits = GENMASK(11, 8), + .sram_pdn_ack_bits = GENMASK(12, 12), + .bp_infracfg = { + BUS_PROT_UPDATE_TOPAXI(MT8167_TOP_AXI_PROT_EN_MM_EMI | + MT8167_TOP_AXI_PROT_EN_MCU_MM), + }, + .caps = MTK_SCPD_ACTIVE_WAKEUP, + }, + [MT8167_POWER_DOMAIN_VDEC] = { + .sta_mask = PWR_STATUS_VDEC, + .ctl_offs = SPM_VDE_PWR_CON, + .sram_pdn_bits = GENMASK(8, 8), + .sram_pdn_ack_bits = GENMASK(12, 12), + .caps = MTK_SCPD_ACTIVE_WAKEUP, + }, + [MT8167_POWER_DOMAIN_ISP] = { + .sta_mask = PWR_STATUS_ISP, + .ctl_offs = SPM_ISP_PWR_CON, + .sram_pdn_bits = GENMASK(11, 8), + .sram_pdn_ack_bits = GENMASK(13, 12), + .caps = MTK_SCPD_ACTIVE_WAKEUP, + }, + [MT8167_POWER_DOMAIN_MFG_ASYNC] = { + .sta_mask = MT8167_PWR_STATUS_MFG_ASYNC, + .ctl_offs = SPM_MFG_ASYNC_PWR_CON, + .sram_pdn_bits = 0, + .sram_pdn_ack_bits = 0, + .bp_infracfg = { + BUS_PROT_UPDATE_TOPAXI(MT8167_TOP_AXI_PROT_EN_MCU_MFG | + MT8167_TOP_AXI_PROT_EN_MFG_EMI), + }, + }, + [MT8167_POWER_DOMAIN_MFG_2D] = { + .sta_mask = MT8167_PWR_STATUS_MFG_2D, + .ctl_offs = SPM_MFG_2D_PWR_CON, + .sram_pdn_bits = GENMASK(11, 8), + .sram_pdn_ack_bits = GENMASK(15, 12), + }, + [MT8167_POWER_DOMAIN_MFG] = { + .sta_mask = PWR_STATUS_MFG, + .ctl_offs = SPM_MFG_PWR_CON, + .sram_pdn_bits = GENMASK(11, 8), + .sram_pdn_ack_bits = GENMASK(15, 12), + }, + [MT8167_POWER_DOMAIN_CONN] = { + .sta_mask = PWR_STATUS_CONN, + .ctl_offs = SPM_CONN_PWR_CON, + .sram_pdn_bits = GENMASK(8, 8), + .sram_pdn_ack_bits = 0, + .caps = MTK_SCPD_ACTIVE_WAKEUP, + .bp_infracfg = { + BUS_PROT_UPDATE_TOPAXI(MT8167_TOP_AXI_PROT_EN_CONN_EMI | + MT8167_TOP_AXI_PROT_EN_CONN_MCU | + MT8167_TOP_AXI_PROT_EN_MCU_CONN), + }, + }, +}; + +static const struct scpsys_soc_data mt8167_scpsys_data = { + .domains_data = scpsys_domain_data_mt8167, + .num_domains = ARRAY_SIZE(scpsys_domain_data_mt8167), + .pwr_sta_offs = SPM_PWR_STATUS, + .pwr_sta2nd_offs = SPM_PWR_STATUS_2ND, +}; + +#endif /* __SOC_MEDIATEK_MT8167_PM_DOMAINS_H */ + diff --git a/drivers/soc/mediatek/mtk-pm-domains.c b/drivers/soc/mediatek/mtk-pm-domains.c index fb70cb3b07b3..2d0d50ff35f0 100644 --- a/drivers/soc/mediatek/mtk-pm-domains.c +++ b/drivers/soc/mediatek/mtk-pm-domains.c @@ -15,6 +15,7 @@ #include #include +#include "mt8167-pm-domains.h" #include "mt8173-pm-domains.h" #include "mt8183-pm-domains.h" #include "mt8192-pm-domains.h" @@ -514,6 +515,10 @@ static void scpsys_domain_cleanup(struct scpsys *scpsys) } static const struct of_device_id scpsys_of_match[] = { + { + .compatible = "mediatek,mt8167-power-controller", + .data = &mt8167_scpsys_data, + }, { .compatible = "mediatek,mt8173-power-controller", .data = &mt8173_scpsys_data, diff --git a/drivers/soc/mediatek/mtk-pm-domains.h b/drivers/soc/mediatek/mtk-pm-domains.h index a2f4d8f97e05..88f5835e1648 100644 --- a/drivers/soc/mediatek/mtk-pm-domains.h +++ b/drivers/soc/mediatek/mtk-pm-domains.h @@ -14,6 +14,7 @@ #define SPM_VEN_PWR_CON 0x0230 #define SPM_ISP_PWR_CON 0x0238 #define SPM_DIS_PWR_CON 0x023c +#define SPM_CONN_PWR_CON 0x0280 #define SPM_VEN2_PWR_CON 0x0298 #define SPM_AUDIO_PWR_CON 0x029c #define SPM_MFG_2D_PWR_CON 0x02c0 diff --git a/include/linux/soc/mediatek/infracfg.h b/include/linux/soc/mediatek/infracfg.h index e7842debc05d..4615a228da51 100644 --- a/include/linux/soc/mediatek/infracfg.h +++ b/include/linux/soc/mediatek/infracfg.h @@ -123,6 +123,14 @@ #define MT8173_TOP_AXI_PROT_EN_MFG_M1 BIT(22) #define MT8173_TOP_AXI_PROT_EN_MFG_SNOOP_OUT BIT(23) +#define MT8167_TOP_AXI_PROT_EN_MM_EMI BIT(1) +#define MT8167_TOP_AXI_PROT_EN_MCU_MFG BIT(2) +#define MT8167_TOP_AXI_PROT_EN_CONN_EMI BIT(4) +#define MT8167_TOP_AXI_PROT_EN_MFG_EMI BIT(5) +#define MT8167_TOP_AXI_PROT_EN_CONN_MCU BIT(8) +#define MT8167_TOP_AXI_PROT_EN_MCU_CONN BIT(9) +#define MT8167_TOP_AXI_PROT_EN_MCU_MM BIT(11) + #define MT2701_TOP_AXI_PROT_EN_MM_M0 BIT(1) #define MT2701_TOP_AXI_PROT_EN_CONN_M BIT(2) #define MT2701_TOP_AXI_PROT_EN_CONN_S BIT(8) -- cgit v1.2.3 From 1570db1da9f57dfbe5dfa3010233c98947497466 Mon Sep 17 00:00:00 2001 From: Chun-Kuang Hu Date: Thu, 3 Dec 2020 07:58:55 +0800 Subject: soc: mediatek: cmdq: Remove cmdq_pkt_flush() rx_callback is a standard mailbox callback mechanism and could cover the function of proprietary cmdq_task_cb, so it is better to use the standard one instead of the proprietary one. But register rx_callback should before mbox_request_channel(), so remove cmdq_pkt_flush() and let client driver implement its own synchronous flush. Signed-off-by: Chun-Kuang Hu Link: https://lore.kernel.org/r/20201202235856.7652-1-chunkuang.hu@kernel.org Signed-off-by: Matthias Brugger --- drivers/soc/mediatek/mtk-cmdq-helper.c | 32 -------------------------------- include/linux/soc/mediatek/mtk-cmdq.h | 12 ------------ 2 files changed, 44 deletions(-) (limited to 'include/linux') diff --git a/drivers/soc/mediatek/mtk-cmdq-helper.c b/drivers/soc/mediatek/mtk-cmdq-helper.c index 280d3bd9f675..3c8e4212d941 100644 --- a/drivers/soc/mediatek/mtk-cmdq-helper.c +++ b/drivers/soc/mediatek/mtk-cmdq-helper.c @@ -463,36 +463,4 @@ int cmdq_pkt_flush_async(struct cmdq_pkt *pkt, cmdq_async_flush_cb cb, } EXPORT_SYMBOL(cmdq_pkt_flush_async); -struct cmdq_flush_completion { - struct completion cmplt; - bool err; -}; - -static void cmdq_pkt_flush_cb(struct cmdq_cb_data data) -{ - struct cmdq_flush_completion *cmplt; - - cmplt = (struct cmdq_flush_completion *)data.data; - if (data.sta != CMDQ_CB_NORMAL) - cmplt->err = true; - else - cmplt->err = false; - complete(&cmplt->cmplt); -} - -int cmdq_pkt_flush(struct cmdq_pkt *pkt) -{ - struct cmdq_flush_completion cmplt; - int err; - - init_completion(&cmplt.cmplt); - err = cmdq_pkt_flush_async(pkt, cmdq_pkt_flush_cb, &cmplt); - if (err < 0) - return err; - wait_for_completion(&cmplt.cmplt); - - return cmplt.err ? -EFAULT : 0; -} -EXPORT_SYMBOL(cmdq_pkt_flush); - MODULE_LICENSE("GPL v2"); diff --git a/include/linux/soc/mediatek/mtk-cmdq.h b/include/linux/soc/mediatek/mtk-cmdq.h index 8e9996610978..ac6b5f3cba95 100644 --- a/include/linux/soc/mediatek/mtk-cmdq.h +++ b/include/linux/soc/mediatek/mtk-cmdq.h @@ -280,16 +280,4 @@ int cmdq_pkt_finalize(struct cmdq_pkt *pkt); int cmdq_pkt_flush_async(struct cmdq_pkt *pkt, cmdq_async_flush_cb cb, void *data); -/** - * cmdq_pkt_flush() - trigger CMDQ to execute the CMDQ packet - * @pkt: the CMDQ packet - * - * Return: 0 for success; else the error code is returned - * - * Trigger CMDQ to execute the CMDQ packet. Note that this is a - * synchronous flush function. When the function returned, the recorded - * commands have been done. - */ -int cmdq_pkt_flush(struct cmdq_pkt *pkt); - #endif /* __MTK_CMDQ_H__ */ -- cgit v1.2.3 From fb02e3ebfb2da27172da47f79a5481c4fe83d751 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 29 Jan 2021 20:51:47 +0100 Subject: staging: hikey9xx: spmi driver: convert to regmap Instead of doing its own SPMI I/O implementation, use the already-existing regmap one. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/7084885f3007ca5daf0d5bc85d038e26ee82dc0d.1611949675.git.mchehab+huawei@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/staging/hikey9xx/Kconfig | 2 + drivers/staging/hikey9xx/hi6421-spmi-pmic.c | 115 ++++++++---------------- drivers/staging/hikey9xx/hi6421v600-regulator.c | 26 +++--- include/linux/mfd/hi6421-spmi-pmic.h | 7 +- 4 files changed, 54 insertions(+), 96 deletions(-) (limited to 'include/linux') diff --git a/drivers/staging/hikey9xx/Kconfig b/drivers/staging/hikey9xx/Kconfig index 2e48ded92a7e..82bb4a22b286 100644 --- a/drivers/staging/hikey9xx/Kconfig +++ b/drivers/staging/hikey9xx/Kconfig @@ -29,6 +29,7 @@ config MFD_HI6421_SPMI depends on OF depends on SPMI select MFD_CORE + select REGMAP_SPMI help Add support for HiSilicon Hi6421v600 SPMI PMIC. Hi6421 includes multi-functions, such as regulators, RTC, codec, Coulomb counter, @@ -44,6 +45,7 @@ config REGULATOR_HI6421V600 tristate "HiSilicon Hi6421v600 PMIC voltage regulator support" depends on MFD_HI6421_SPMI && OF depends on REGULATOR + select REGMAP help This driver provides support for the voltage regulators on HiSilicon Hi6421v600 PMU / Codec IC. diff --git a/drivers/staging/hikey9xx/hi6421-spmi-pmic.c b/drivers/staging/hikey9xx/hi6421-spmi-pmic.c index 69570876f93e..3d612bd46231 100644 --- a/drivers/staging/hikey9xx/hi6421-spmi-pmic.c +++ b/drivers/staging/hikey9xx/hi6421-spmi-pmic.c @@ -41,81 +41,22 @@ static const struct mfd_cell hi6421v600_devs[] = { { .name = "hi6421v600-regulator", }, }; -/* - * The PMIC register is only 8-bit. - * Hisilicon SoC use hardware to map PMIC register into SoC mapping. - * At here, we are accessing SoC register with 32-bit. - */ -int hi6421_spmi_pmic_read(struct hi6421_spmi_pmic *pmic, int reg) +static irqreturn_t hi6421_spmi_irq_handler(int irq, void *priv) { - struct spmi_device *pdev; - u8 read_value = 0; - u32 ret; - - pdev = to_spmi_device(pmic->dev); - if (!pdev) { - pr_err("%s: pdev get failed!\n", __func__); - return -ENODEV; - } - - ret = spmi_ext_register_readl(pdev, reg, &read_value, 1); - if (ret) { - pr_err("%s: spmi_ext_register_readl failed!\n", __func__); - return ret; - } - return read_value; -} -EXPORT_SYMBOL(hi6421_spmi_pmic_read); - -int hi6421_spmi_pmic_write(struct hi6421_spmi_pmic *pmic, int reg, u32 val) -{ - struct spmi_device *pdev; - u32 ret; - - pdev = to_spmi_device(pmic->dev); - if (!pdev) { - pr_err("%s: pdev get failed!\n", __func__); - return -ENODEV; - } - - ret = spmi_ext_register_writel(pdev, reg, (unsigned char *)&val, 1); - if (ret) - pr_err("%s: spmi_ext_register_writel failed!\n", __func__); - - return ret; -} -EXPORT_SYMBOL(hi6421_spmi_pmic_write); - -int hi6421_spmi_pmic_rmw(struct hi6421_spmi_pmic *pmic, int reg, - u32 mask, u32 bits) -{ - unsigned long flags; - u32 data; - int ret; - - spin_lock_irqsave(&pmic->lock, flags); - data = hi6421_spmi_pmic_read(pmic, reg) & ~mask; - data |= mask & bits; - ret = hi6421_spmi_pmic_write(pmic, reg, data); - spin_unlock_irqrestore(&pmic->lock, flags); - - return ret; -} -EXPORT_SYMBOL(hi6421_spmi_pmic_rmw); - -static irqreturn_t hi6421_spmi_irq_handler(int irq, void *data) -{ - struct hi6421_spmi_pmic *pmic = (struct hi6421_spmi_pmic *)data; + struct hi6421_spmi_pmic *pmic = (struct hi6421_spmi_pmic *)priv; unsigned long pending; + unsigned int data; int i, offset; for (i = 0; i < HISI_IRQ_ARRAY; i++) { - pending = hi6421_spmi_pmic_read(pmic, (i + SOC_PMIC_IRQ0_ADDR)); - pending &= HISI_MASK_FIELD; - if (pending != 0) - pr_debug("pending[%d]=0x%lx\n\r", i, pending); + regmap_read(pmic->map, offset, &data); + data &= HISI_MASK_FIELD; + if (data != 0) + pr_debug("data[%d]=0x%d\n\r", i, data); + regmap_write(pmic->map, i + SOC_PMIC_IRQ0_ADDR, data); - hi6421_spmi_pmic_write(pmic, (i + SOC_PMIC_IRQ0_ADDR), pending); + /* for_each_set_bit() macro requires unsigned long */ + pending = data; /* solve powerkey order */ if ((i == HISI_IRQ_KEY_NUM) && @@ -137,16 +78,18 @@ static irqreturn_t hi6421_spmi_irq_handler(int irq, void *data) static void hi6421_spmi_irq_mask(struct irq_data *d) { struct hi6421_spmi_pmic *pmic = irq_data_get_irq_chip_data(d); - u32 data, offset; unsigned long flags; + unsigned int data; + u32 offset; offset = (irqd_to_hwirq(d) >> 3); offset += SOC_PMIC_IRQ_MASK_0_ADDR; spin_lock_irqsave(&pmic->lock, flags); - data = hi6421_spmi_pmic_read(pmic, offset); + + regmap_read(pmic->map, offset, &data); data |= (1 << (irqd_to_hwirq(d) & 0x07)); - hi6421_spmi_pmic_write(pmic, offset, data); + regmap_write(pmic->map, offset, data); spin_unlock_irqrestore(&pmic->lock, flags); } @@ -160,9 +103,9 @@ static void hi6421_spmi_irq_unmask(struct irq_data *d) offset += SOC_PMIC_IRQ_MASK_0_ADDR; spin_lock_irqsave(&pmic->lock, flags); - data = hi6421_spmi_pmic_read(pmic, offset); + regmap_read(pmic->map, offset, &data); data &= ~(1 << (irqd_to_hwirq(d) & 0x07)); - hi6421_spmi_pmic_write(pmic, offset, data); + regmap_write(pmic->map, offset, data); spin_unlock_irqrestore(&pmic->lock, flags); } @@ -194,27 +137,36 @@ static const struct irq_domain_ops hi6421_spmi_domain_ops = { static void hi6421_spmi_pmic_irq_prc(struct hi6421_spmi_pmic *pmic) { - int i, pending; + int i; + unsigned int pending; for (i = 0 ; i < HISI_IRQ_ARRAY; i++) - hi6421_spmi_pmic_write(pmic, SOC_PMIC_IRQ_MASK_0_ADDR + i, + regmap_write(pmic->map, SOC_PMIC_IRQ_MASK_0_ADDR + i, HISI_MASK_STATE); for (i = 0 ; i < HISI_IRQ_ARRAY; i++) { - pending = hi6421_spmi_pmic_read(pmic, SOC_PMIC_IRQ0_ADDR + i); + regmap_read(pmic->map, SOC_PMIC_IRQ0_ADDR + i, &pending); pr_debug("PMU IRQ address value:irq[0x%x] = 0x%x\n", SOC_PMIC_IRQ0_ADDR + i, pending); - hi6421_spmi_pmic_write(pmic, SOC_PMIC_IRQ0_ADDR + i, - HISI_MASK_STATE); + regmap_write(pmic->map, SOC_PMIC_IRQ0_ADDR + i, + HISI_MASK_STATE); } } +static const struct regmap_config spmi_regmap_config = { + .reg_bits = 16, + .val_bits = 8, + .max_register = 0xffff, + .fast_io = true +}; + static int hi6421_spmi_pmic_probe(struct spmi_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; struct hi6421_spmi_pmic *pmic; + struct regmap *map; unsigned int virq; int ret, i; @@ -222,9 +174,14 @@ static int hi6421_spmi_pmic_probe(struct spmi_device *pdev) if (!pmic) return -ENOMEM; + map = devm_regmap_init_spmi_ext(pdev, &spmi_regmap_config); + if (IS_ERR(map)) + return PTR_ERR(map); + spin_lock_init(&pmic->lock); pmic->dev = dev; + pmic->map = map; pmic->gpio = of_get_gpio(np, 0); if (pmic->gpio < 0) diff --git a/drivers/staging/hikey9xx/hi6421v600-regulator.c b/drivers/staging/hikey9xx/hi6421v600-regulator.c index 382a0b21643e..9e319fa11137 100644 --- a/drivers/staging/hikey9xx/hi6421v600-regulator.c +++ b/drivers/staging/hikey9xx/hi6421v600-regulator.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -100,7 +101,7 @@ static int hi6421_spmi_regulator_is_enabled(struct regulator_dev *rdev) struct hi6421_spmi_pmic *pmic = sreg->pmic; u32 reg_val; - reg_val = hi6421_spmi_pmic_read(pmic, rdev->desc->enable_reg); + regmap_read(pmic->map, rdev->desc->enable_reg, ®_val); return ((reg_val & rdev->desc->enable_mask) != 0); } @@ -114,9 +115,9 @@ static int hi6421_spmi_regulator_enable(struct regulator_dev *rdev) /* cannot enable more than one regulator at one time */ mutex_lock(&sreg->enable_mutex); - ret = hi6421_spmi_pmic_rmw(pmic, rdev->desc->enable_reg, - rdev->desc->enable_mask, - rdev->desc->enable_mask); + ret = regmap_update_bits(pmic->map, rdev->desc->enable_reg, + rdev->desc->enable_mask, + rdev->desc->enable_mask); /* Avoid powering up multiple devices at the same time */ usleep_range(rdev->desc->off_on_delay, rdev->desc->off_on_delay + 60); @@ -131,8 +132,8 @@ static int hi6421_spmi_regulator_disable(struct regulator_dev *rdev) struct hi6421_spmi_reg_info *sreg = rdev_get_drvdata(rdev); struct hi6421_spmi_pmic *pmic = sreg->pmic; - return hi6421_spmi_pmic_rmw(pmic, rdev->desc->enable_reg, - rdev->desc->enable_mask, 0); + return regmap_update_bits(pmic->map, rdev->desc->enable_reg, + rdev->desc->enable_mask, 0); } static int hi6421_spmi_regulator_get_voltage_sel(struct regulator_dev *rdev) @@ -141,7 +142,7 @@ static int hi6421_spmi_regulator_get_voltage_sel(struct regulator_dev *rdev) struct hi6421_spmi_pmic *pmic = sreg->pmic; u32 reg_val; - reg_val = hi6421_spmi_pmic_read(pmic, rdev->desc->vsel_reg); + regmap_read(pmic->map, rdev->desc->vsel_reg, ®_val); return (reg_val & rdev->desc->vsel_mask) >> (ffs(rdev->desc->vsel_mask) - 1); } @@ -159,8 +160,8 @@ static int hi6421_spmi_regulator_set_voltage_sel(struct regulator_dev *rdev, reg_val = selector << (ffs(rdev->desc->vsel_mask) - 1); /* set voltage selector */ - return hi6421_spmi_pmic_rmw(pmic, rdev->desc->vsel_reg, - rdev->desc->vsel_mask, reg_val); + return regmap_update_bits(pmic->map, rdev->desc->vsel_reg, + rdev->desc->vsel_mask, reg_val); } static unsigned int hi6421_spmi_regulator_get_mode(struct regulator_dev *rdev) @@ -169,7 +170,7 @@ static unsigned int hi6421_spmi_regulator_get_mode(struct regulator_dev *rdev) struct hi6421_spmi_pmic *pmic = sreg->pmic; u32 reg_val; - reg_val = hi6421_spmi_pmic_read(pmic, rdev->desc->enable_reg); + regmap_read(pmic->map, rdev->desc->enable_reg, ®_val); if (reg_val & sreg->eco_mode_mask) return REGULATOR_MODE_IDLE; @@ -195,8 +196,8 @@ static int hi6421_spmi_regulator_set_mode(struct regulator_dev *rdev, return -EINVAL; } - return hi6421_spmi_pmic_rmw(pmic, rdev->desc->enable_reg, - sreg->eco_mode_mask, val); + return regmap_update_bits(pmic->map, rdev->desc->enable_reg, + sreg->eco_mode_mask, val); } static unsigned int @@ -304,6 +305,7 @@ static int hi6421_spmi_regulator_probe(struct platform_device *pdev) config.dev = pdev->dev.parent; config.driver_data = sreg; + config.regmap = pmic->map; rdev = devm_regulator_register(dev, &info->desc, &config); if (IS_ERR(rdev)) { diff --git a/include/linux/mfd/hi6421-spmi-pmic.h b/include/linux/mfd/hi6421-spmi-pmic.h index 0c2214612c4e..c5a4eac1fdc0 100644 --- a/include/linux/mfd/hi6421-spmi-pmic.h +++ b/include/linux/mfd/hi6421-spmi-pmic.h @@ -12,6 +12,7 @@ #define __HISI_PMIC_H #include +#include #define HISI_ECO_MODE_ENABLE (1) #define HISI_ECO_MODE_DISABLE (0) @@ -25,13 +26,9 @@ struct hi6421_spmi_pmic { int irq; int gpio; unsigned int *irqs; + struct regmap *map; }; -int hi6421_spmi_pmic_read(struct hi6421_spmi_pmic *pmic, int reg); -int hi6421_spmi_pmic_write(struct hi6421_spmi_pmic *pmic, int reg, u32 val); -int hi6421_spmi_pmic_rmw(struct hi6421_spmi_pmic *pmic, int reg, - u32 mask, u32 bits); - enum hi6421_spmi_pmic_irq_list { OTMP = 0, VBUS_CONNECT, -- cgit v1.2.3 From fcd732406c5d65ba92515af81ca281a4f8348687 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 29 Jan 2021 20:51:49 +0100 Subject: staging: hikey9xx: hi6421-spmi-pmic: rename some vars - When referring to regmap, rename map to regmap - inside hi6421-spmi-pmic, call private data struct as ddata. No functional changes. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/d23592b11ac606e3b9a3ff95a754cb75921e60aa.1611949675.git.mchehab+huawei@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/staging/hikey9xx/hi6421-spmi-pmic.c | 104 ++++++++++++------------ drivers/staging/hikey9xx/hi6421v600-regulator.c | 10 +-- include/linux/mfd/hi6421-spmi-pmic.h | 2 +- 3 files changed, 58 insertions(+), 58 deletions(-) (limited to 'include/linux') diff --git a/drivers/staging/hikey9xx/hi6421-spmi-pmic.c b/drivers/staging/hikey9xx/hi6421-spmi-pmic.c index 3d612bd46231..a4ffeb06ed6a 100644 --- a/drivers/staging/hikey9xx/hi6421-spmi-pmic.c +++ b/drivers/staging/hikey9xx/hi6421-spmi-pmic.c @@ -43,17 +43,17 @@ static const struct mfd_cell hi6421v600_devs[] = { static irqreturn_t hi6421_spmi_irq_handler(int irq, void *priv) { - struct hi6421_spmi_pmic *pmic = (struct hi6421_spmi_pmic *)priv; + struct hi6421_spmi_pmic *ddata = (struct hi6421_spmi_pmic *)priv; unsigned long pending; unsigned int data; int i, offset; for (i = 0; i < HISI_IRQ_ARRAY; i++) { - regmap_read(pmic->map, offset, &data); + regmap_read(ddata->regmap, offset, &data); data &= HISI_MASK_FIELD; if (data != 0) pr_debug("data[%d]=0x%d\n\r", i, data); - regmap_write(pmic->map, i + SOC_PMIC_IRQ0_ADDR, data); + regmap_write(ddata->regmap, i + SOC_PMIC_IRQ0_ADDR, data); /* for_each_set_bit() macro requires unsigned long */ pending = data; @@ -61,14 +61,14 @@ static irqreturn_t hi6421_spmi_irq_handler(int irq, void *priv) /* solve powerkey order */ if ((i == HISI_IRQ_KEY_NUM) && ((pending & HISI_IRQ_KEY_VALUE) == HISI_IRQ_KEY_VALUE)) { - generic_handle_irq(pmic->irqs[HISI_IRQ_KEY_DOWN]); - generic_handle_irq(pmic->irqs[HISI_IRQ_KEY_UP]); + generic_handle_irq(ddata->irqs[HISI_IRQ_KEY_DOWN]); + generic_handle_irq(ddata->irqs[HISI_IRQ_KEY_UP]); pending &= (~HISI_IRQ_KEY_VALUE); } if (pending) { for_each_set_bit(offset, &pending, HISI_BITS) - generic_handle_irq(pmic->irqs[offset + i * HISI_BITS]); + generic_handle_irq(ddata->irqs[offset + i * HISI_BITS]); } } @@ -77,7 +77,7 @@ static irqreturn_t hi6421_spmi_irq_handler(int irq, void *priv) static void hi6421_spmi_irq_mask(struct irq_data *d) { - struct hi6421_spmi_pmic *pmic = irq_data_get_irq_chip_data(d); + struct hi6421_spmi_pmic *ddata = irq_data_get_irq_chip_data(d); unsigned long flags; unsigned int data; u32 offset; @@ -85,28 +85,28 @@ static void hi6421_spmi_irq_mask(struct irq_data *d) offset = (irqd_to_hwirq(d) >> 3); offset += SOC_PMIC_IRQ_MASK_0_ADDR; - spin_lock_irqsave(&pmic->lock, flags); + spin_lock_irqsave(&ddata->lock, flags); - regmap_read(pmic->map, offset, &data); + regmap_read(ddata->regmap, offset, &data); data |= (1 << (irqd_to_hwirq(d) & 0x07)); - regmap_write(pmic->map, offset, data); - spin_unlock_irqrestore(&pmic->lock, flags); + regmap_write(ddata->regmap, offset, data); + spin_unlock_irqrestore(&ddata->lock, flags); } static void hi6421_spmi_irq_unmask(struct irq_data *d) { - struct hi6421_spmi_pmic *pmic = irq_data_get_irq_chip_data(d); + struct hi6421_spmi_pmic *ddata = irq_data_get_irq_chip_data(d); u32 data, offset; unsigned long flags; offset = (irqd_to_hwirq(d) >> 3); offset += SOC_PMIC_IRQ_MASK_0_ADDR; - spin_lock_irqsave(&pmic->lock, flags); - regmap_read(pmic->map, offset, &data); + spin_lock_irqsave(&ddata->lock, flags); + regmap_read(ddata->regmap, offset, &data); data &= ~(1 << (irqd_to_hwirq(d) & 0x07)); - regmap_write(pmic->map, offset, data); - spin_unlock_irqrestore(&pmic->lock, flags); + regmap_write(ddata->regmap, offset, data); + spin_unlock_irqrestore(&ddata->lock, flags); } static struct irq_chip hi6421_spmi_pmu_irqchip = { @@ -120,11 +120,11 @@ static struct irq_chip hi6421_spmi_pmu_irqchip = { static int hi6421_spmi_irq_map(struct irq_domain *d, unsigned int virq, irq_hw_number_t hw) { - struct hi6421_spmi_pmic *pmic = d->host_data; + struct hi6421_spmi_pmic *ddata = d->host_data; irq_set_chip_and_handler_name(virq, &hi6421_spmi_pmu_irqchip, handle_simple_irq, "hisi"); - irq_set_chip_data(virq, pmic); + irq_set_chip_data(virq, ddata); irq_set_irq_type(virq, IRQ_TYPE_NONE); return 0; @@ -135,21 +135,21 @@ static const struct irq_domain_ops hi6421_spmi_domain_ops = { .xlate = irq_domain_xlate_twocell, }; -static void hi6421_spmi_pmic_irq_prc(struct hi6421_spmi_pmic *pmic) +static void hi6421_spmi_pmic_irq_prc(struct hi6421_spmi_pmic *ddata) { int i; unsigned int pending; for (i = 0 ; i < HISI_IRQ_ARRAY; i++) - regmap_write(pmic->map, SOC_PMIC_IRQ_MASK_0_ADDR + i, + regmap_write(ddata->regmap, SOC_PMIC_IRQ_MASK_0_ADDR + i, HISI_MASK_STATE); for (i = 0 ; i < HISI_IRQ_ARRAY; i++) { - regmap_read(pmic->map, SOC_PMIC_IRQ0_ADDR + i, &pending); + regmap_read(ddata->regmap, SOC_PMIC_IRQ0_ADDR + i, &pending); pr_debug("PMU IRQ address value:irq[0x%x] = 0x%x\n", SOC_PMIC_IRQ0_ADDR + i, pending); - regmap_write(pmic->map, SOC_PMIC_IRQ0_ADDR + i, + regmap_write(ddata->regmap, SOC_PMIC_IRQ0_ADDR + i, HISI_MASK_STATE); } } @@ -165,79 +165,79 @@ static int hi6421_spmi_pmic_probe(struct spmi_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; - struct hi6421_spmi_pmic *pmic; + struct hi6421_spmi_pmic *ddata; struct regmap *map; unsigned int virq; int ret, i; - pmic = devm_kzalloc(dev, sizeof(*pmic), GFP_KERNEL); - if (!pmic) + ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL); + if (!ddata) return -ENOMEM; map = devm_regmap_init_spmi_ext(pdev, &spmi_regmap_config); if (IS_ERR(map)) return PTR_ERR(map); - spin_lock_init(&pmic->lock); + spin_lock_init(&ddata->lock); - pmic->dev = dev; - pmic->map = map; + ddata->dev = dev; + ddata->regmap = map; - pmic->gpio = of_get_gpio(np, 0); - if (pmic->gpio < 0) - return pmic->gpio; + ddata->gpio = of_get_gpio(np, 0); + if (ddata->gpio < 0) + return ddata->gpio; - if (!gpio_is_valid(pmic->gpio)) + if (!gpio_is_valid(ddata->gpio)) return -EINVAL; - ret = devm_gpio_request_one(dev, pmic->gpio, GPIOF_IN, "pmic"); + ret = devm_gpio_request_one(dev, ddata->gpio, GPIOF_IN, "pmic"); if (ret < 0) { - dev_err(dev, "failed to request gpio%d\n", pmic->gpio); + dev_err(dev, "failed to request gpio%d\n", ddata->gpio); return ret; } - pmic->irq = gpio_to_irq(pmic->gpio); + ddata->irq = gpio_to_irq(ddata->gpio); - hi6421_spmi_pmic_irq_prc(pmic); + hi6421_spmi_pmic_irq_prc(ddata); - pmic->irqs = devm_kzalloc(dev, HISI_IRQ_NUM * sizeof(int), GFP_KERNEL); - if (!pmic->irqs) { + ddata->irqs = devm_kzalloc(dev, HISI_IRQ_NUM * sizeof(int), GFP_KERNEL); + if (!ddata->irqs) { ret = -ENOMEM; goto irq_malloc; } - pmic->domain = irq_domain_add_simple(np, HISI_IRQ_NUM, 0, - &hi6421_spmi_domain_ops, pmic); - if (!pmic->domain) { + ddata->domain = irq_domain_add_simple(np, HISI_IRQ_NUM, 0, + &hi6421_spmi_domain_ops, ddata); + if (!ddata->domain) { dev_err(dev, "failed irq domain add simple!\n"); ret = -ENODEV; goto irq_malloc; } for (i = 0; i < HISI_IRQ_NUM; i++) { - virq = irq_create_mapping(pmic->domain, i); + virq = irq_create_mapping(ddata->domain, i); if (!virq) { dev_err(dev, "Failed mapping hwirq\n"); ret = -ENOSPC; goto irq_malloc; } - pmic->irqs[i] = virq; - dev_dbg(dev, "%s: pmic->irqs[%d] = %d\n", - __func__, i, pmic->irqs[i]); + ddata->irqs[i] = virq; + dev_dbg(dev, "%s: ddata->irqs[%d] = %d\n", + __func__, i, ddata->irqs[i]); } - ret = request_threaded_irq(pmic->irq, hi6421_spmi_irq_handler, NULL, + ret = request_threaded_irq(ddata->irq, hi6421_spmi_irq_handler, NULL, IRQF_TRIGGER_LOW | IRQF_SHARED | IRQF_NO_SUSPEND, - "pmic", pmic); + "pmic", ddata); if (ret < 0) { dev_err(dev, "could not claim pmic IRQ: error %d\n", ret); goto irq_malloc; } - dev_set_drvdata(&pdev->dev, pmic); + dev_set_drvdata(&pdev->dev, ddata); /* - * The logic below will rely that the pmic is already stored at + * The logic below will rely that the ddata is already stored at * drvdata. */ dev_dbg(&pdev->dev, "SPMI-PMIC: adding children for %pOF\n", @@ -251,16 +251,16 @@ static int hi6421_spmi_pmic_probe(struct spmi_device *pdev) dev_err(dev, "Failed to add child devices: %d\n", ret); irq_malloc: - free_irq(pmic->irq, pmic); + free_irq(ddata->irq, ddata); return ret; } static void hi6421_spmi_pmic_remove(struct spmi_device *pdev) { - struct hi6421_spmi_pmic *pmic = dev_get_drvdata(&pdev->dev); + struct hi6421_spmi_pmic *ddata = dev_get_drvdata(&pdev->dev); - free_irq(pmic->irq, pmic); + free_irq(ddata->irq, ddata); } static const struct of_device_id pmic_spmi_id_table[] = { diff --git a/drivers/staging/hikey9xx/hi6421v600-regulator.c b/drivers/staging/hikey9xx/hi6421v600-regulator.c index 7090107b9ec2..c801bb840962 100644 --- a/drivers/staging/hikey9xx/hi6421v600-regulator.c +++ b/drivers/staging/hikey9xx/hi6421v600-regulator.c @@ -104,7 +104,7 @@ static int hi6421_spmi_regulator_enable(struct regulator_dev *rdev) /* cannot enable more than one regulator at one time */ mutex_lock(&sreg->enable_mutex); - ret = regmap_update_bits(pmic->map, rdev->desc->enable_reg, + ret = regmap_update_bits(pmic->regmap, rdev->desc->enable_reg, rdev->desc->enable_mask, rdev->desc->enable_mask); @@ -121,7 +121,7 @@ static int hi6421_spmi_regulator_disable(struct regulator_dev *rdev) struct hi6421_spmi_reg_info *sreg = rdev_get_drvdata(rdev); struct hi6421_spmi_pmic *pmic = sreg->pmic; - return regmap_update_bits(pmic->map, rdev->desc->enable_reg, + return regmap_update_bits(pmic->regmap, rdev->desc->enable_reg, rdev->desc->enable_mask, 0); } @@ -131,7 +131,7 @@ static unsigned int hi6421_spmi_regulator_get_mode(struct regulator_dev *rdev) struct hi6421_spmi_pmic *pmic = sreg->pmic; u32 reg_val; - regmap_read(pmic->map, rdev->desc->enable_reg, ®_val); + regmap_read(pmic->regmap, rdev->desc->enable_reg, ®_val); if (reg_val & sreg->eco_mode_mask) return REGULATOR_MODE_IDLE; @@ -157,7 +157,7 @@ static int hi6421_spmi_regulator_set_mode(struct regulator_dev *rdev, return -EINVAL; } - return regmap_update_bits(pmic->map, rdev->desc->enable_reg, + return regmap_update_bits(pmic->regmap, rdev->desc->enable_reg, sreg->eco_mode_mask, val); } @@ -266,7 +266,7 @@ static int hi6421_spmi_regulator_probe(struct platform_device *pdev) config.dev = pdev->dev.parent; config.driver_data = sreg; - config.regmap = pmic->map; + config.regmap = pmic->regmap; rdev = devm_regulator_register(dev, &info->desc, &config); if (IS_ERR(rdev)) { diff --git a/include/linux/mfd/hi6421-spmi-pmic.h b/include/linux/mfd/hi6421-spmi-pmic.h index c5a4eac1fdc0..aa8d5382f559 100644 --- a/include/linux/mfd/hi6421-spmi-pmic.h +++ b/include/linux/mfd/hi6421-spmi-pmic.h @@ -26,7 +26,7 @@ struct hi6421_spmi_pmic { int irq; int gpio; unsigned int *irqs; - struct regmap *map; + struct regmap *regmap; }; enum hi6421_spmi_pmic_irq_list { -- cgit v1.2.3 From 9d8dbe989029e6d767ed56773ac65409da625381 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 29 Jan 2021 20:51:51 +0100 Subject: staging: hikey9xx: hi6421-spmi-pmic: cleanup header file Remove the IRQ list from the header, as this is used only inside the driver itself. Also, get rid of two unused defines. The net result is that only struct hi6421_spmi_pmic remains on it, as this is used by the regulator driver. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/138c3a11e4de0ebabdf27932957852136c2f7510.1611949675.git.mchehab+huawei@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/staging/hikey9xx/hi6421-spmi-pmic.c | 17 +++++++++++++++++ include/linux/mfd/hi6421-spmi-pmic.h | 20 -------------------- 2 files changed, 17 insertions(+), 20 deletions(-) (limited to 'include/linux') diff --git a/drivers/staging/hikey9xx/hi6421-spmi-pmic.c b/drivers/staging/hikey9xx/hi6421-spmi-pmic.c index c8e55b7b08e2..909f7b106af4 100644 --- a/drivers/staging/hikey9xx/hi6421-spmi-pmic.c +++ b/drivers/staging/hikey9xx/hi6421-spmi-pmic.c @@ -17,6 +17,23 @@ #include #include +enum hi6421_spmi_pmic_irq_list { + OTMP = 0, + VBUS_CONNECT, + VBUS_DISCONNECT, + ALARMON_R, + HOLD_6S, + HOLD_1S, + POWERKEY_UP, + POWERKEY_DOWN, + OCP_SCP_R, + COUL_R, + SIM0_HPD_R, + SIM0_HPD_F, + SIM1_HPD_R, + SIM1_HPD_F, + PMIC_IRQ_LIST_MAX, +}; /* 8-bit register offset in PMIC */ #define HISI_MASK_STATE 0xff diff --git a/include/linux/mfd/hi6421-spmi-pmic.h b/include/linux/mfd/hi6421-spmi-pmic.h index aa8d5382f559..4d61cb266a18 100644 --- a/include/linux/mfd/hi6421-spmi-pmic.h +++ b/include/linux/mfd/hi6421-spmi-pmic.h @@ -14,9 +14,6 @@ #include #include -#define HISI_ECO_MODE_ENABLE (1) -#define HISI_ECO_MODE_DISABLE (0) - struct hi6421_spmi_pmic { struct resource *res; struct device *dev; @@ -29,21 +26,4 @@ struct hi6421_spmi_pmic { struct regmap *regmap; }; -enum hi6421_spmi_pmic_irq_list { - OTMP = 0, - VBUS_CONNECT, - VBUS_DISCONNECT, - ALARMON_R, - HOLD_6S, - HOLD_1S, - POWERKEY_UP, - POWERKEY_DOWN, - OCP_SCP_R, - COUL_R, - SIM0_HPD_R, - SIM0_HPD_F, - SIM1_HPD_R, - SIM1_HPD_F, - PMIC_IRQ_LIST_MAX, -}; #endif /* __HISI_PMIC_H */ -- cgit v1.2.3 From fb7ba1870d5fdf45513fc04af8e46492eb65a5e3 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 29 Jan 2021 20:51:55 +0100 Subject: staging: hikey9xx: hi6421-spmi-pmic: update copyright notes At PMIC subsystem, C89 comments are preferred over C99. While here, also update the copyrights of the header file. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/5a86478c8ccb93d3105485b5f16e20e9c12e2196.1611949675.git.mchehab+huawei@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/staging/hikey9xx/hi6421-spmi-pmic.c | 14 +++++++------- include/linux/mfd/hi6421-spmi-pmic.h | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/drivers/staging/hikey9xx/hi6421-spmi-pmic.c b/drivers/staging/hikey9xx/hi6421-spmi-pmic.c index 9c10f7c4e7c9..2301f4fcd48d 100644 --- a/drivers/staging/hikey9xx/hi6421-spmi-pmic.c +++ b/drivers/staging/hikey9xx/hi6421-spmi-pmic.c @@ -1,11 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 -// -// Device driver for regulators in HISI PMIC IC -// -// Copyright (c) 2013 Linaro Ltd. -// Copyright (c) 2011 Hisilicon. -// -// Copyright (c) 2020-2021 Huawei Technologies Co., Ltd +/* + * Device driver for regulators in HISI PMIC IC + * + * Copyright (c) 2013 Linaro Ltd. + * Copyright (c) 2011 Hisilicon. + * Copyright (c) 2020-2021 Huawei Technologies Co., Ltd + */ #include #include diff --git a/include/linux/mfd/hi6421-spmi-pmic.h b/include/linux/mfd/hi6421-spmi-pmic.h index 4d61cb266a18..2660226138b8 100644 --- a/include/linux/mfd/hi6421-spmi-pmic.h +++ b/include/linux/mfd/hi6421-spmi-pmic.h @@ -4,6 +4,7 @@ * * Copyright (c) 2013 Linaro Ltd. * Copyright (C) 2011 Hisilicon. + * Copyright (c) 2020-2021 Huawei Technologies Co., Ltd * * Guodong Xu */ -- cgit v1.2.3 From f5030e252687be6e999bd52feb1f79d515b2f684 Mon Sep 17 00:00:00 2001 From: Benson Leung Date: Thu, 28 Jan 2021 22:14:02 -0800 Subject: usb: typec: Provide PD Specification Revision for cable and partner The USB Power Delivery specification Section 6.2.1.1.5 outlines revision backward compatibility requirements starting from Revision 3.0. The Port, the Cable Plug, and the Port Partner may support either revision 2 or revision 3 independently, and communication between ports, partners, and cables of different revisions are allowed under rules that the parties agree to communicate between each other using the lowest common operating revision. This may mean that Port-to-Partner operating revision comms may be different than Port-to-CablePlug operating revision comms. For example, it is possible for a R3.0 port to communicate with a R3.0 partner using R3.0 messages, while the R3.0 port (in the same session) must communicate with the R2.0 cable using R2.0 messages only. Introduce individual revision number properties for cable and port partner so that the port can track them independently. Reviewed-by: Heikki Krogerus Signed-off-by: Benson Leung Link: https://lore.kernel.org/r/20210129061406.2680146-3-bleung@chromium.org Signed-off-by: Greg Kroah-Hartman --- Documentation/ABI/testing/sysfs-class-typec | 13 +++++++++++++ drivers/usb/typec/class.c | 30 +++++++++++++++++++++++++---- include/linux/usb/typec.h | 10 ++++++++++ 3 files changed, 49 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-class-typec b/Documentation/ABI/testing/sysfs-class-typec index b61480535fdc..40122d915ae1 100644 --- a/Documentation/ABI/testing/sysfs-class-typec +++ b/Documentation/ABI/testing/sysfs-class-typec @@ -112,6 +112,19 @@ Description: - "3.0": USB Power Delivery Release 3.0 - "3.1": USB Power Delivery Release 3.1 +What: /sys/class/typec/-{partner|cable}/usb_power_delivery_revision +Date: January 2021 +Contact: Benson Leung +Description: + Revision number of the supported USB Power Delivery + specification of the port partner or cable, or 0.0 when USB + Power Delivery is not supported. + + Example values: + - "2.0": USB Power Delivery Release 2.0 + - "3.0": USB Power Delivery Release 3.0 + - "3.1": USB Power Delivery Release 3.1 + What: /sys/class/typec//usb_typec_revision Date: April 2017 Contact: Heikki Krogerus diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index 4f60ee7ba76a..b5241f4756c2 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -27,6 +27,7 @@ struct typec_cable { enum typec_plug_type type; struct usb_pd_identity *identity; unsigned int active:1; + u16 pd_revision; /* 0300H = "3.0" */ }; struct typec_partner { @@ -36,6 +37,7 @@ struct typec_partner { enum typec_accessory accessory; struct ida mode_ids; int num_altmodes; + u16 pd_revision; /* 0300H = "3.0" */ }; struct typec_port { @@ -264,6 +266,11 @@ type_show(struct device *dev, struct device_attribute *attr, char *buf) } static DEVICE_ATTR_RO(type); +static ssize_t usb_power_delivery_revision_show(struct device *dev, + struct device_attribute *attr, + char *buf); +static DEVICE_ATTR_RO(usb_power_delivery_revision); + /* ------------------------------------------------------------------------- */ /* Alternate Modes */ @@ -680,6 +687,7 @@ static struct attribute *typec_partner_attrs[] = { &dev_attr_supports_usb_power_delivery.attr, &dev_attr_number_of_alternate_modes.attr, &dev_attr_type.attr, + &dev_attr_usb_power_delivery_revision.attr, NULL }; @@ -815,6 +823,7 @@ struct typec_partner *typec_register_partner(struct typec_port *port, partner->usb_pd = desc->usb_pd; partner->accessory = desc->accessory; partner->num_altmodes = -1; + partner->pd_revision = desc->pd_revision; if (desc->identity) { /* @@ -1028,6 +1037,7 @@ static DEVICE_ATTR_RO(plug_type); static struct attribute *typec_cable_attrs[] = { &dev_attr_type.attr, &dev_attr_plug_type.attr, + &dev_attr_usb_power_delivery_revision.attr, NULL }; ATTRIBUTE_GROUPS(typec_cable); @@ -1130,6 +1140,7 @@ struct typec_cable *typec_register_cable(struct typec_port *port, cable->type = desc->type; cable->active = desc->active; + cable->pd_revision = desc->pd_revision; if (desc->identity) { /* @@ -1499,12 +1510,23 @@ static ssize_t usb_power_delivery_revision_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct typec_port *p = to_typec_port(dev); - u16 rev = p->cap->pd_revision; + u16 rev = 0; - return sprintf(buf, "%d.%d\n", (rev >> 8) & 0xff, (rev >> 4) & 0xf); + if (is_typec_partner(dev)) { + struct typec_partner *partner = to_typec_partner(dev); + + rev = partner->pd_revision; + } else if (is_typec_cable(dev)) { + struct typec_cable *cable = to_typec_cable(dev); + + rev = cable->pd_revision; + } else if (is_typec_port(dev)) { + struct typec_port *p = to_typec_port(dev); + + rev = p->cap->pd_revision; + } + return sysfs_emit(buf, "%d.%d\n", (rev >> 8) & 0xff, (rev >> 4) & 0xf); } -static DEVICE_ATTR_RO(usb_power_delivery_revision); static ssize_t orientation_show(struct device *dev, struct device_attribute *attr, diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h index 54475323f83b..42c6b7c07a99 100644 --- a/include/linux/usb/typec.h +++ b/include/linux/usb/typec.h @@ -164,6 +164,7 @@ struct typec_plug_desc { * @type: The plug type from USB PD Cable VDO * @active: Is the cable active or passive * @identity: Result of Discover Identity command + * @pd_revision: USB Power Delivery Specification revision if supported * * Represents USB Type-C Cable attached to USB Type-C port. */ @@ -171,6 +172,8 @@ struct typec_cable_desc { enum typec_plug_type type; unsigned int active:1; struct usb_pd_identity *identity; + u16 pd_revision; /* 0300H = "3.0" */ + }; /* @@ -178,15 +181,22 @@ struct typec_cable_desc { * @usb_pd: USB Power Delivery support * @accessory: Audio, Debug or none. * @identity: Discover Identity command data + * @pd_revision: USB Power Delivery Specification Revision if supported * * Details about a partner that is attached to USB Type-C port. If @identity * member exists when partner is registered, a directory named "identity" is * created to sysfs for the partner device. + * + * @pd_revision is based on the setting of the "Specification Revision" field + * in the message header on the initial "Source Capabilities" message received + * from the partner, or a "Request" message received from the partner, depending + * on whether our port is a Sink or a Source. */ struct typec_partner_desc { unsigned int usb_pd:1; enum typec_accessory accessory; struct usb_pd_identity *identity; + u16 pd_revision; /* 0300H = "3.0" */ }; /** -- cgit v1.2.3 From 29b01295a829fba7399ee84afff4e64660e49f04 Mon Sep 17 00:00:00 2001 From: Benson Leung Date: Thu, 28 Jan 2021 22:14:03 -0800 Subject: usb: typec: Add typec_partner_set_pd_revision The partner's PD revision may be resolved later than the port partner registration since the port partner creation may take place once Type-C detects the port has changed state, but before PD communication is completed. Add a setter so that the partner's PD revision can be attached to it once it becomes available. If the revision is set to a valid version (not 0), the setter will also refresh the partner's usb_pd flag and notify on "supports_usb_power_delivery" sysfs property as well. Reviewed-by: Heikki Krogerus Signed-off-by: Benson Leung Link: https://lore.kernel.org/r/20210129061406.2680146-4-bleung@chromium.org Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/class.c | 30 ++++++++++++++++++++++++++++++ include/linux/usb/typec.h | 1 + 2 files changed, 31 insertions(+) (limited to 'include/linux') diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index b5241f4756c2..b6ceab3dc16b 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -748,6 +748,36 @@ int typec_partner_set_identity(struct typec_partner *partner) } EXPORT_SYMBOL_GPL(typec_partner_set_identity); +/** + * typec_partner_set_pd_revision - Set the PD revision supported by the partner + * @partner: The partner to be updated. + * @pd_revision: USB Power Delivery Specification Revision supported by partner + * + * This routine is used to report that the PD revision of the port partner has + * become available. + * + * Returns 0 on success or negative error number on failure. + */ +int typec_partner_set_pd_revision(struct typec_partner *partner, u16 pd_revision) +{ + int ret; + + if (partner->pd_revision == pd_revision) + return 0; + + partner->pd_revision = pd_revision; + sysfs_notify(&partner->dev.kobj, NULL, "usb_power_delivery_revision"); + if (pd_revision != 0 && !partner->usb_pd) { + partner->usb_pd = 1; + sysfs_notify(&partner->dev.kobj, NULL, + "supports_usb_power_delivery"); + } + kobject_uevent(&partner->dev.kobj, KOBJ_CHANGE); + + return 0; +} +EXPORT_SYMBOL_GPL(typec_partner_set_pd_revision); + /** * typec_partner_set_num_altmodes - Set the number of available partner altmodes * @partner: The partner to be updated. diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h index 42c6b7c07a99..4946eca742d5 100644 --- a/include/linux/usb/typec.h +++ b/include/linux/usb/typec.h @@ -126,6 +126,7 @@ struct typec_altmode_desc { enum typec_port_data roles; }; +int typec_partner_set_pd_revision(struct typec_partner *partner, u16 pd_revision); int typec_partner_set_num_altmodes(struct typec_partner *partner, int num_altmodes); struct typec_altmode *typec_partner_register_altmode(struct typec_partner *partner, -- cgit v1.2.3 From 2a6c6b7d7ad346f0679d0963cb19b3f0ea7ef32c Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Thu, 28 Jan 2021 14:40:07 -0800 Subject: perf/core: Add PERF_SAMPLE_WEIGHT_STRUCT Current PERF_SAMPLE_WEIGHT sample type is very useful to expresses the cost of an action represented by the sample. This allows the profiler to scale the samples to be more informative to the programmer. It could also help to locate a hotspot, e.g., when profiling by memory latencies, the expensive load appear higher up in the histograms. But current PERF_SAMPLE_WEIGHT sample type is solely determined by one factor. This could be a problem, if users want two or more factors to contribute to the weight. For example, Golden Cove core PMU can provide both the instruction latency and the cache Latency information as factors for the memory profiling. For current X86 platforms, although meminfo::latency is defined as a u64, only the lower 32 bits include the valid data in practice (No memory access could last than 4G cycles). The higher 32 bits can be used to store new factors. Add a new sample type, PERF_SAMPLE_WEIGHT_STRUCT, to indicate the new sample weight structure. It shares the same space as the PERF_SAMPLE_WEIGHT sample type. Users can apply either the PERF_SAMPLE_WEIGHT sample type or the PERF_SAMPLE_WEIGHT_STRUCT sample type to retrieve the sample weight, but they cannot apply both sample types simultaneously. Currently, only X86 and PowerPC use the PERF_SAMPLE_WEIGHT sample type. - For PowerPC, there is nothing changed for the PERF_SAMPLE_WEIGHT sample type. There is no effect for the new PERF_SAMPLE_WEIGHT_STRUCT sample type. PowerPC can re-struct the weight field similarly later. - For X86, the same value will be dumped for the PERF_SAMPLE_WEIGHT sample type or the PERF_SAMPLE_WEIGHT_STRUCT sample type for now. The following patches will apply the new factors for the PERF_SAMPLE_WEIGHT_STRUCT sample type. The field in the union perf_sample_weight should be shared among different architectures. A generic name is required, but it's hard to abstract a name that applies to all architectures. For example, on X86, the fields are to store all kinds of latency. While on PowerPC, it stores MMCRA[TECX/TECM], which should not be latency. So a general name prefix 'var$NUM' is used here. Suggested-by: Peter Zijlstra (Intel) Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/1611873611-156687-2-git-send-email-kan.liang@linux.intel.com --- arch/powerpc/perf/core-book3s.c | 2 +- arch/x86/events/intel/ds.c | 17 +++++++++-------- include/linux/perf_event.h | 4 ++-- include/uapi/linux/perf_event.h | 42 +++++++++++++++++++++++++++++++++++++++-- kernel/events/core.c | 11 +++++++---- 5 files changed, 59 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 28206b1fe172..869d999a836e 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -2195,7 +2195,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, if (event->attr.sample_type & PERF_SAMPLE_WEIGHT && ppmu->get_mem_weight) - ppmu->get_mem_weight(&data.weight); + ppmu->get_mem_weight(&data.weight.full); if (perf_event_overflow(event, &data, regs)) power_pmu_stop(event, 0); diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 67dbc91bccfe..2f54b1fbb895 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -960,7 +960,8 @@ static void adaptive_pebs_record_size_update(void) } #define PERF_PEBS_MEMINFO_TYPE (PERF_SAMPLE_ADDR | PERF_SAMPLE_DATA_SRC | \ - PERF_SAMPLE_PHYS_ADDR | PERF_SAMPLE_WEIGHT | \ + PERF_SAMPLE_PHYS_ADDR | \ + PERF_SAMPLE_WEIGHT_TYPE | \ PERF_SAMPLE_TRANSACTION | \ PERF_SAMPLE_DATA_PAGE_SIZE) @@ -987,7 +988,7 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event) gprs = (sample_type & PERF_SAMPLE_REGS_INTR) && (attr->sample_regs_intr & PEBS_GP_REGS); - tsx_weight = (sample_type & PERF_SAMPLE_WEIGHT) && + tsx_weight = (sample_type & PERF_SAMPLE_WEIGHT_TYPE) && ((attr->config & INTEL_ARCH_EVENT_MASK) == x86_pmu.rtm_abort_event); @@ -1369,8 +1370,8 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event, /* * Use latency for weight (only avail with PEBS-LL) */ - if (fll && (sample_type & PERF_SAMPLE_WEIGHT)) - data->weight = pebs->lat; + if (fll && (sample_type & PERF_SAMPLE_WEIGHT_TYPE)) + data->weight.full = pebs->lat; /* * data.data_src encodes the data source @@ -1462,8 +1463,8 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event, if (x86_pmu.intel_cap.pebs_format >= 2) { /* Only set the TSX weight when no memory weight. */ - if ((sample_type & PERF_SAMPLE_WEIGHT) && !fll) - data->weight = intel_get_tsx_weight(pebs->tsx_tuning); + if ((sample_type & PERF_SAMPLE_WEIGHT_TYPE) && !fll) + data->weight.full = intel_get_tsx_weight(pebs->tsx_tuning); if (sample_type & PERF_SAMPLE_TRANSACTION) data->txn = intel_get_tsx_transaction(pebs->tsx_tuning, @@ -1577,8 +1578,8 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, } if (format_size & PEBS_DATACFG_MEMINFO) { - if (sample_type & PERF_SAMPLE_WEIGHT) - data->weight = meminfo->latency ?: + if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) + data->weight.full = meminfo->latency ?: intel_get_tsx_weight(meminfo->tsx_tuning); if (sample_type & PERF_SAMPLE_DATA_SRC) diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 9a38f579bc76..fab42cfbd350 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -998,7 +998,7 @@ struct perf_sample_data { struct perf_raw_record *raw; struct perf_branch_stack *br_stack; u64 period; - u64 weight; + union perf_sample_weight weight; u64 txn; union perf_mem_data_src data_src; @@ -1047,7 +1047,7 @@ static inline void perf_sample_data_init(struct perf_sample_data *data, data->raw = NULL; data->br_stack = NULL; data->period = period; - data->weight = 0; + data->weight.full = 0; data->data_src.val = PERF_MEM_NA; data->txn = 0; } diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h index b15e3447cd9f..b2cc246ec119 100644 --- a/include/uapi/linux/perf_event.h +++ b/include/uapi/linux/perf_event.h @@ -145,12 +145,14 @@ enum perf_event_sample_format { PERF_SAMPLE_CGROUP = 1U << 21, PERF_SAMPLE_DATA_PAGE_SIZE = 1U << 22, PERF_SAMPLE_CODE_PAGE_SIZE = 1U << 23, + PERF_SAMPLE_WEIGHT_STRUCT = 1U << 24, - PERF_SAMPLE_MAX = 1U << 24, /* non-ABI */ + PERF_SAMPLE_MAX = 1U << 25, /* non-ABI */ __PERF_SAMPLE_CALLCHAIN_EARLY = 1ULL << 63, /* non-ABI; internal use */ }; +#define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT | PERF_SAMPLE_WEIGHT_STRUCT) /* * values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set * @@ -890,7 +892,24 @@ enum perf_event_type { * char data[size]; * u64 dyn_size; } && PERF_SAMPLE_STACK_USER * - * { u64 weight; } && PERF_SAMPLE_WEIGHT + * { union perf_sample_weight + * { + * u64 full; && PERF_SAMPLE_WEIGHT + * #if defined(__LITTLE_ENDIAN_BITFIELD) + * struct { + * u32 var1_dw; + * u16 var2_w; + * u16 var3_w; + * } && PERF_SAMPLE_WEIGHT_STRUCT + * #elif defined(__BIG_ENDIAN_BITFIELD) + * struct { + * u16 var3_w; + * u16 var2_w; + * u32 var1_dw; + * } && PERF_SAMPLE_WEIGHT_STRUCT + * #endif + * } + * } * { u64 data_src; } && PERF_SAMPLE_DATA_SRC * { u64 transaction; } && PERF_SAMPLE_TRANSACTION * { u64 abi; # enum perf_sample_regs_abi @@ -1248,4 +1267,23 @@ struct perf_branch_entry { reserved:40; }; +union perf_sample_weight { + __u64 full; +#if defined(__LITTLE_ENDIAN_BITFIELD) + struct { + __u32 var1_dw; + __u16 var2_w; + __u16 var3_w; + }; +#elif defined(__BIG_ENDIAN_BITFIELD) + struct { + __u16 var3_w; + __u16 var2_w; + __u32 var1_dw; + }; +#else +#error "Unknown endianness" +#endif +}; + #endif /* _UAPI_LINUX_PERF_EVENT_H */ diff --git a/kernel/events/core.c b/kernel/events/core.c index 55d18791a72d..5206097d4d3d 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -1879,8 +1879,8 @@ static void __perf_event_header_size(struct perf_event *event, u64 sample_type) if (sample_type & PERF_SAMPLE_PERIOD) size += sizeof(data->period); - if (sample_type & PERF_SAMPLE_WEIGHT) - size += sizeof(data->weight); + if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) + size += sizeof(data->weight.full); if (sample_type & PERF_SAMPLE_READ) size += event->read_size; @@ -6907,8 +6907,8 @@ void perf_output_sample(struct perf_output_handle *handle, data->regs_user.regs); } - if (sample_type & PERF_SAMPLE_WEIGHT) - perf_output_put(handle, data->weight); + if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) + perf_output_put(handle, data->weight.full); if (sample_type & PERF_SAMPLE_DATA_SRC) perf_output_put(handle, data->data_src.val); @@ -11564,6 +11564,9 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, if (attr->sample_type & PERF_SAMPLE_CGROUP) return -EINVAL; #endif + if ((attr->sample_type & PERF_SAMPLE_WEIGHT) && + (attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) + return -EINVAL; out: return ret; -- cgit v1.2.3 From 8dd5cada393f6f4e825833a6ff05b1f51f36a791 Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Mon, 18 Jan 2021 03:55:18 +0300 Subject: opp: Add dev_pm_opp_find_level_ceil() Add a ceil version of the dev_pm_opp_find_level(). It's handy to have if levels don't start from 0 in OPP table and zero usually means a minimal level. Tested-by: Peter Geis Tested-by: Nicolas Chauvet Tested-by: Matt Merhar Signed-off-by: Dmitry Osipenko Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/pm_opp.h | 8 ++++++++ 2 files changed, 57 insertions(+) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index dc7a298f3611..b29f31146770 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -449,6 +449,55 @@ struct dev_pm_opp *dev_pm_opp_find_level_exact(struct device *dev, } EXPORT_SYMBOL_GPL(dev_pm_opp_find_level_exact); +/** + * dev_pm_opp_find_level_ceil() - search for an rounded up level + * @dev: device for which we do this operation + * @level: level to search for + * + * Return: Searches for rounded up match in the opp table and returns pointer + * to the matching opp if found, else returns ERR_PTR in case of error and + * should be handled using IS_ERR. Error return values can be: + * EINVAL: for bad pointer + * ERANGE: no match found for search + * ENODEV: if device not found in list of registered devices + * + * The callers are required to call dev_pm_opp_put() for the returned OPP after + * use. + */ +struct dev_pm_opp *dev_pm_opp_find_level_ceil(struct device *dev, + unsigned int *level) +{ + struct opp_table *opp_table; + struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE); + + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) { + int r = PTR_ERR(opp_table); + + dev_err(dev, "%s: OPP table not found (%d)\n", __func__, r); + return ERR_PTR(r); + } + + mutex_lock(&opp_table->lock); + + list_for_each_entry(temp_opp, &opp_table->opp_list, node) { + if (temp_opp->available && temp_opp->level >= *level) { + opp = temp_opp; + *level = opp->level; + + /* Increment the reference count of OPP */ + dev_pm_opp_get(opp); + break; + } + } + + mutex_unlock(&opp_table->lock); + dev_pm_opp_put_opp_table(opp_table); + + return opp; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_find_level_ceil); + static noinline struct dev_pm_opp *_find_freq_ceil(struct opp_table *opp_table, unsigned long *freq) { diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 1435c054016a..2b3030cb2ed2 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -111,6 +111,8 @@ struct dev_pm_opp *dev_pm_opp_find_freq_exact(struct device *dev, bool available); struct dev_pm_opp *dev_pm_opp_find_level_exact(struct device *dev, unsigned int level); +struct dev_pm_opp *dev_pm_opp_find_level_ceil(struct device *dev, + unsigned int *level); struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev, unsigned long *freq); @@ -226,6 +228,12 @@ static inline struct dev_pm_opp *dev_pm_opp_find_level_exact(struct device *dev, return ERR_PTR(-ENOTSUPP); } +static inline struct dev_pm_opp *dev_pm_opp_find_level_ceil(struct device *dev, + unsigned int *level) +{ + return ERR_PTR(-ENOTSUPP); +} + static inline struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev, unsigned long *freq) { -- cgit v1.2.3 From 597ff5431fd41afa888809f7936508a15c977cde Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Mon, 18 Jan 2021 03:55:19 +0300 Subject: opp: Add dev_pm_opp_get_required_pstate() Add dev_pm_opp_get_required_pstate() which allows OPP users to retrieve required performance state of a given OPP. Tested-by: Peter Geis Tested-by: Nicolas Chauvet Tested-by: Matt Merhar Signed-off-by: Dmitry Osipenko Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 22 ++++++++++++++++++++++ include/linux/pm_opp.h | 10 ++++++++++ 2 files changed, 32 insertions(+) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index b29f31146770..64a95aaa90eb 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -145,6 +145,28 @@ unsigned int dev_pm_opp_get_level(struct dev_pm_opp *opp) } EXPORT_SYMBOL_GPL(dev_pm_opp_get_level); +/** + * dev_pm_opp_get_required_pstate() - Gets the required performance state + * corresponding to an available opp + * @opp: opp for which performance state has to be returned for + * @index: index of the required opp + * + * Return: performance state read from device tree corresponding to the + * required opp, else return 0. + */ +unsigned int dev_pm_opp_get_required_pstate(struct dev_pm_opp *opp, + unsigned int index) +{ + if (IS_ERR_OR_NULL(opp) || !opp->available || + index >= opp->opp_table->required_opp_count) { + pr_err("%s: Invalid parameters\n", __func__); + return 0; + } + + return opp->required_opps[index]->pstate; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_required_pstate); + /** * dev_pm_opp_is_turbo() - Returns if opp is turbo OPP or not * @opp: opp for which turbo mode is being verified diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 2b3030cb2ed2..8f926815bad9 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -98,6 +98,9 @@ unsigned long dev_pm_opp_get_freq(struct dev_pm_opp *opp); unsigned int dev_pm_opp_get_level(struct dev_pm_opp *opp); +unsigned int dev_pm_opp_get_required_pstate(struct dev_pm_opp *opp, + unsigned int index); + bool dev_pm_opp_is_turbo(struct dev_pm_opp *opp); int dev_pm_opp_get_opp_count(struct device *dev); @@ -186,6 +189,13 @@ static inline unsigned int dev_pm_opp_get_level(struct dev_pm_opp *opp) return 0; } +static inline +unsigned int dev_pm_opp_get_required_pstate(struct dev_pm_opp *opp, + unsigned int index) +{ + return 0; +} + static inline bool dev_pm_opp_is_turbo(struct dev_pm_opp *opp) { return false; -- cgit v1.2.3 From ce8073d83f63a2cdcfc1b86d769456726faad51d Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Thu, 21 Jan 2021 01:26:47 +0300 Subject: opp: Add dev_pm_opp_sync_regulators() Extend OPP API with dev_pm_opp_sync_regulators() function, which syncs voltage state of regulators. Tested-by: Peter Geis Tested-by: Nicolas Chauvet Tested-by: Matt Merhar Signed-off-by: Dmitry Osipenko [ Viresh: Added unlikely() ] Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 41 +++++++++++++++++++++++++++++++++++++++++ include/linux/pm_opp.h | 6 ++++++ 2 files changed, 47 insertions(+) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 64a95aaa90eb..bf7cdab0ba64 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2584,3 +2584,44 @@ void dev_pm_opp_remove_table(struct device *dev) dev_pm_opp_put_opp_table(opp_table); } EXPORT_SYMBOL_GPL(dev_pm_opp_remove_table); + +/** + * dev_pm_opp_sync_regulators() - Sync state of voltage regulators + * @dev: device for which we do this operation + * + * Sync voltage state of the OPP table regulators. + * + * Return: 0 on success or a negative error value. + */ +int dev_pm_opp_sync_regulators(struct device *dev) +{ + struct opp_table *opp_table; + struct regulator *reg; + int i, ret = 0; + + /* Device may not have OPP table */ + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) + return 0; + + /* Regulator may not be required for the device */ + if (unlikely(!opp_table->regulators)) + goto put_table; + + /* Nothing to sync if voltage wasn't changed */ + if (!opp_table->enabled) + goto put_table; + + for (i = 0; i < opp_table->regulator_count; i++) { + reg = opp_table->regulators[i]; + ret = regulator_sync_voltage(reg); + if (ret) + break; + } +put_table: + /* Drop reference taken by _find_opp_table() */ + dev_pm_opp_put_opp_table(opp_table); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_sync_regulators); diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 8f926815bad9..979b208bc4a8 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -161,6 +161,7 @@ int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, const struct cpumask *cp int dev_pm_opp_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask); void dev_pm_opp_remove_table(struct device *dev); void dev_pm_opp_cpumask_remove_table(const struct cpumask *cpumask); +int dev_pm_opp_sync_regulators(struct device *dev); #else static inline struct opp_table *dev_pm_opp_get_opp_table(struct device *dev) { @@ -384,6 +385,11 @@ static inline void dev_pm_opp_cpumask_remove_table(const struct cpumask *cpumask { } +static inline int dev_pm_opp_sync_regulators(struct device *dev) +{ + return -ENOTSUPP; +} + #endif /* CONFIG_PM_OPP */ #if defined(CONFIG_PM_OPP) && defined(CONFIG_OF) -- cgit v1.2.3 From 559fef0dfd91145b59b7c61061504f344ecf9ad8 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 27 Jan 2021 14:23:45 +0530 Subject: opp: Add dev_pm_opp_of_add_table_noclk() A few drivers have device's clk but they don't want the OPP core to handle that. Add a new helper for them, dev_pm_opp_of_add_table_noclk(). Signed-off-by: Viresh Kumar Tested-by: Dmitry Osipenko --- drivers/opp/of.c | 18 ++++++++++++++++++ include/linux/pm_opp.h | 6 ++++++ 2 files changed, 24 insertions(+) (limited to 'include/linux') diff --git a/drivers/opp/of.c b/drivers/opp/of.c index d4b51b2e384f..a905497c75f8 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -1030,6 +1030,24 @@ int dev_pm_opp_of_add_table_indexed(struct device *dev, int index) } EXPORT_SYMBOL_GPL(dev_pm_opp_of_add_table_indexed); +/** + * dev_pm_opp_of_add_table_noclk() - Initialize indexed opp table from device + * tree without getting clk for device. + * @dev: device pointer used to lookup OPP table. + * @index: Index number. + * + * Register the initial OPP table with the OPP library for given device only + * using the "operating-points-v2" property. Do not try to get the clk for the + * device. + * + * Return: Refer to dev_pm_opp_of_add_table() for return values. + */ +int dev_pm_opp_of_add_table_noclk(struct device *dev, int index) +{ + return _of_add_table_indexed(dev, index, false); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_of_add_table_noclk); + /* CPU device specific helpers */ /** diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 979b208bc4a8..158158620dde 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -395,6 +395,7 @@ static inline int dev_pm_opp_sync_regulators(struct device *dev) #if defined(CONFIG_PM_OPP) && defined(CONFIG_OF) int dev_pm_opp_of_add_table(struct device *dev); int dev_pm_opp_of_add_table_indexed(struct device *dev, int index); +int dev_pm_opp_of_add_table_noclk(struct device *dev, int index); void dev_pm_opp_of_remove_table(struct device *dev); int dev_pm_opp_of_cpumask_add_table(const struct cpumask *cpumask); void dev_pm_opp_of_cpumask_remove_table(const struct cpumask *cpumask); @@ -419,6 +420,11 @@ static inline int dev_pm_opp_of_add_table_indexed(struct device *dev, int index) return -ENOTSUPP; } +static inline int dev_pm_opp_of_add_table_noclk(struct device *dev, int index) +{ + return -ENOTSUPP; +} + static inline void dev_pm_opp_of_remove_table(struct device *dev) { } -- cgit v1.2.3 From a3c47af6942dc8e07a4328913d0263a965786895 Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Mon, 18 Jan 2021 03:55:20 +0300 Subject: opp: Add devm_pm_opp_register_set_opp_helper Add resource-managed version of dev_pm_opp_register_set_opp_helper(). Tested-by: Peter Geis Tested-by: Nicolas Chauvet Tested-by: Matt Merhar Signed-off-by: Dmitry Osipenko [ Viresh: Manually apply the patch and relocate the routines ] Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 34 ++++++++++++++++++++++++++++++++++ include/linux/pm_opp.h | 8 ++++++++ 2 files changed, 42 insertions(+) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 52f4a64926e6..09069a564896 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2106,6 +2106,40 @@ void dev_pm_opp_unregister_set_opp_helper(struct opp_table *opp_table) } EXPORT_SYMBOL_GPL(dev_pm_opp_unregister_set_opp_helper); +static void devm_pm_opp_unregister_set_opp_helper(void *data) +{ + dev_pm_opp_unregister_set_opp_helper(data); +} + +/** + * devm_pm_opp_register_set_opp_helper() - Register custom set OPP helper + * @dev: Device for which the helper is getting registered. + * @set_opp: Custom set OPP helper. + * + * This is a resource-managed version of dev_pm_opp_register_set_opp_helper(). + * + * Return: pointer to 'struct opp_table' on success and errorno otherwise. + */ +struct opp_table * +devm_pm_opp_register_set_opp_helper(struct device *dev, + int (*set_opp)(struct dev_pm_set_opp_data *data)) +{ + struct opp_table *opp_table; + int err; + + opp_table = dev_pm_opp_register_set_opp_helper(dev, set_opp); + if (IS_ERR(opp_table)) + return opp_table; + + err = devm_add_action_or_reset(dev, devm_pm_opp_unregister_set_opp_helper, + opp_table); + if (err) + return ERR_PTR(err); + + return opp_table; +} +EXPORT_SYMBOL_GPL(devm_pm_opp_register_set_opp_helper); + static void _opp_detach_genpd(struct opp_table *opp_table) { int index; diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 158158620dde..473daf34160d 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -152,6 +152,7 @@ struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char * name); void dev_pm_opp_put_clkname(struct opp_table *opp_table); struct opp_table *dev_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)); void dev_pm_opp_unregister_set_opp_helper(struct opp_table *opp_table); +struct opp_table *devm_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)); struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, const char **names, struct device ***virt_devs); void dev_pm_opp_detach_genpd(struct opp_table *opp_table); int dev_pm_opp_xlate_performance_state(struct opp_table *src_table, struct opp_table *dst_table, unsigned int pstate); @@ -324,6 +325,13 @@ static inline struct opp_table *dev_pm_opp_register_set_opp_helper(struct device static inline void dev_pm_opp_unregister_set_opp_helper(struct opp_table *opp_table) {} +static inline struct opp_table * +devm_pm_opp_register_set_opp_helper(struct device *dev, + int (*set_opp)(struct dev_pm_set_opp_data *data)) +{ + return ERR_PTR(-ENOTSUPP); +} + static inline struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name) { return ERR_PTR(-ENOTSUPP); -- cgit v1.2.3 From b4b9e223eccaeec6e05d927c292d4425fd18f243 Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Mon, 18 Jan 2021 03:55:21 +0300 Subject: opp: Add devm_pm_opp_attach_genpd Add resource-managed version of dev_pm_opp_attach_genpd(). Signed-off-by: Dmitry Osipenko [ Viresh: Manually apply the patch and relocate the routines ] Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 36 ++++++++++++++++++++++++++++++++++++ include/linux/pm_opp.h | 7 +++++++ 2 files changed, 43 insertions(+) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 09069a564896..ce0ec5bde22a 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2275,6 +2275,42 @@ void dev_pm_opp_detach_genpd(struct opp_table *opp_table) } EXPORT_SYMBOL_GPL(dev_pm_opp_detach_genpd); +static void devm_pm_opp_detach_genpd(void *data) +{ + dev_pm_opp_detach_genpd(data); +} + +/** + * devm_pm_opp_attach_genpd - Attach genpd(s) for the device and save virtual + * device pointer + * @dev: Consumer device for which the genpd is getting attached. + * @names: Null terminated array of pointers containing names of genpd to attach. + * @virt_devs: Pointer to return the array of virtual devices. + * + * This is a resource-managed version of dev_pm_opp_attach_genpd(). + * + * Return: pointer to 'struct opp_table' on success and errorno otherwise. + */ +struct opp_table * +devm_pm_opp_attach_genpd(struct device *dev, const char **names, + struct device ***virt_devs) +{ + struct opp_table *opp_table; + int err; + + opp_table = dev_pm_opp_attach_genpd(dev, names, virt_devs); + if (IS_ERR(opp_table)) + return opp_table; + + err = devm_add_action_or_reset(dev, devm_pm_opp_detach_genpd, + opp_table); + if (err) + return ERR_PTR(err); + + return opp_table; +} +EXPORT_SYMBOL_GPL(devm_pm_opp_attach_genpd); + /** * dev_pm_opp_xlate_performance_state() - Find required OPP's pstate for src_table. * @src_table: OPP table which has dst_table as one of its required OPP table. diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 473daf34160d..a2c871799603 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -155,6 +155,7 @@ void dev_pm_opp_unregister_set_opp_helper(struct opp_table *opp_table); struct opp_table *devm_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)); struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, const char **names, struct device ***virt_devs); void dev_pm_opp_detach_genpd(struct opp_table *opp_table); +struct opp_table *devm_pm_opp_attach_genpd(struct device *dev, const char **names, struct device ***virt_devs); int dev_pm_opp_xlate_performance_state(struct opp_table *src_table, struct opp_table *dst_table, unsigned int pstate); int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq); int dev_pm_opp_set_bw(struct device *dev, struct dev_pm_opp *opp); @@ -360,6 +361,12 @@ static inline struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, cons static inline void dev_pm_opp_detach_genpd(struct opp_table *opp_table) {} +static inline struct opp_table *devm_pm_opp_attach_genpd(struct device *dev, + const char **names, struct device ***virt_devs) +{ + return ERR_PTR(-ENOTSUPP); +} + static inline int dev_pm_opp_xlate_performance_state(struct opp_table *src_table, struct opp_table *dst_table, unsigned int pstate) { return -ENOTSUPP; -- cgit v1.2.3 From abbe348340c7df9e08fd7c24491c1be31ab65370 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 21 Jan 2021 12:15:36 +0530 Subject: opp: Implement dev_pm_opp_set_opp() The new helper dev_pm_opp_set_opp() can be used for configuring the devices for a particular OPP and can be used by different type of devices, even the ones which don't change frequency (like power domains). Signed-off-by: Viresh Kumar Tested-by: Dmitry Osipenko --- drivers/opp/core.c | 28 ++++++++++++++++++++++++++++ include/linux/pm_opp.h | 6 ++++++ 2 files changed, 34 insertions(+) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 2b5584ef0350..fac84d5a1d45 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -1136,6 +1136,34 @@ put_opp_table: } EXPORT_SYMBOL_GPL(dev_pm_opp_set_rate); +/** + * dev_pm_opp_set_opp() - Configure device for OPP + * @dev: device for which we do this operation + * @opp: OPP to set to + * + * This configures the device based on the properties of the OPP passed to this + * routine. + * + * Return: 0 on success, a negative error number otherwise. + */ +int dev_pm_opp_set_opp(struct device *dev, struct dev_pm_opp *opp) +{ + struct opp_table *opp_table; + int ret; + + opp_table = _find_opp_table(dev); + if (IS_ERR(opp_table)) { + dev_err(dev, "%s: device opp doesn't exist\n", __func__); + return PTR_ERR(opp_table); + } + + ret = _set_opp(dev, opp_table, opp, opp ? opp->rate : 0); + dev_pm_opp_put_opp_table(opp_table); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_set_opp); + /* OPP-dev Helpers */ static void _remove_opp_dev(struct opp_device *opp_dev, struct opp_table *opp_table) diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index a2c871799603..7b1d47ab3fb3 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -158,6 +158,7 @@ void dev_pm_opp_detach_genpd(struct opp_table *opp_table); struct opp_table *devm_pm_opp_attach_genpd(struct device *dev, const char **names, struct device ***virt_devs); int dev_pm_opp_xlate_performance_state(struct opp_table *src_table, struct opp_table *dst_table, unsigned int pstate); int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq); +int dev_pm_opp_set_opp(struct device *dev, struct dev_pm_opp *opp); int dev_pm_opp_set_bw(struct device *dev, struct dev_pm_opp *opp); int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, const struct cpumask *cpumask); int dev_pm_opp_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask); @@ -377,6 +378,11 @@ static inline int dev_pm_opp_set_rate(struct device *dev, unsigned long target_f return -ENOTSUPP; } +static inline int dev_pm_opp_set_opp(struct device *dev, struct dev_pm_opp *opp) +{ + return -ENOTSUPP; +} + static inline int dev_pm_opp_set_bw(struct device *dev, struct dev_pm_opp *opp) { return -EOPNOTSUPP; -- cgit v1.2.3 From 240ae50e23061cd1fe1937daab195c17226ffd2e Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 21 Jan 2021 15:27:55 +0530 Subject: opp: Remove dev_pm_opp_set_bw() All the users have migrated to dev_pm_opp_set_opp() now, get rid of the duplicate API, dev_pm_opp_set_bw(), which only performs a part of the new API. While at it, remove the unnecessary parameter to _set_opp_bw(). Signed-off-by: Viresh Kumar Tested-by: Dmitry Osipenko --- drivers/opp/core.c | 41 +++++------------------------------------ include/linux/pm_opp.h | 6 ------ 2 files changed, 5 insertions(+), 42 deletions(-) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index fac84d5a1d45..6958a5cd2fd8 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -798,7 +798,7 @@ restore_voltage: } static int _set_opp_bw(const struct opp_table *opp_table, - struct dev_pm_opp *opp, struct device *dev, bool remove) + struct dev_pm_opp *opp, struct device *dev) { u32 avg, peak; int i, ret; @@ -807,7 +807,7 @@ static int _set_opp_bw(const struct opp_table *opp_table, return 0; for (i = 0; i < opp_table->path_count; i++) { - if (remove) { + if (!opp) { avg = 0; peak = 0; } else { @@ -817,7 +817,7 @@ static int _set_opp_bw(const struct opp_table *opp_table, ret = icc_set_bw(opp_table->paths[i], avg, peak); if (ret) { dev_err(dev, "Failed to %s bandwidth[%d]: %d\n", - remove ? "remove" : "set", i, ret); + opp ? "set" : "remove", i, ret); return ret; } } @@ -917,37 +917,6 @@ static int _set_required_opps(struct device *dev, return ret; } -/** - * dev_pm_opp_set_bw() - sets bandwidth levels corresponding to an opp - * @dev: device for which we do this operation - * @opp: opp based on which the bandwidth levels are to be configured - * - * This configures the bandwidth to the levels specified by the OPP. However - * if the OPP specified is NULL the bandwidth levels are cleared out. - * - * Return: 0 on success or a negative error value. - */ -int dev_pm_opp_set_bw(struct device *dev, struct dev_pm_opp *opp) -{ - struct opp_table *opp_table; - int ret; - - opp_table = _find_opp_table(dev); - if (IS_ERR(opp_table)) { - dev_err(dev, "%s: device opp table doesn't exist\n", __func__); - return PTR_ERR(opp_table); - } - - if (opp) - ret = _set_opp_bw(opp_table, opp, dev, false); - else - ret = _set_opp_bw(opp_table, NULL, dev, true); - - dev_pm_opp_put_opp_table(opp_table); - return ret; -} -EXPORT_SYMBOL_GPL(dev_pm_opp_set_bw); - static void _find_current_opp(struct device *dev, struct opp_table *opp_table) { struct dev_pm_opp *opp = ERR_PTR(-ENODEV); @@ -988,7 +957,7 @@ static int _disable_opp_table(struct device *dev, struct opp_table *opp_table) if (!_get_opp_count(opp_table)) return 0; - ret = _set_opp_bw(opp_table, NULL, dev, true); + ret = _set_opp_bw(opp_table, NULL, dev); if (ret) return ret; @@ -1056,7 +1025,7 @@ static int _set_opp(struct device *dev, struct opp_table *opp_table, } if (!ret) { - ret = _set_opp_bw(opp_table, opp, dev, false); + ret = _set_opp_bw(opp_table, opp, dev); if (!ret) { opp_table->enabled = true; dev_pm_opp_put(old_opp); diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 7b1d47ab3fb3..25e47ab937b9 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -159,7 +159,6 @@ struct opp_table *devm_pm_opp_attach_genpd(struct device *dev, const char **name int dev_pm_opp_xlate_performance_state(struct opp_table *src_table, struct opp_table *dst_table, unsigned int pstate); int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq); int dev_pm_opp_set_opp(struct device *dev, struct dev_pm_opp *opp); -int dev_pm_opp_set_bw(struct device *dev, struct dev_pm_opp *opp); int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, const struct cpumask *cpumask); int dev_pm_opp_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask); void dev_pm_opp_remove_table(struct device *dev); @@ -383,11 +382,6 @@ static inline int dev_pm_opp_set_opp(struct device *dev, struct dev_pm_opp *opp) return -ENOTSUPP; } -static inline int dev_pm_opp_set_bw(struct device *dev, struct dev_pm_opp *opp) -{ - return -EOPNOTSUPP; -} - static inline int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, const struct cpumask *cpumask) { return -ENOTSUPP; -- cgit v1.2.3 From f3988bc5d58b768c5cf0dadf5f0e49f7176432df Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 1 Feb 2021 10:35:07 +0530 Subject: opp: Fix "foo * bar" should be "foo *bar" Fix checkpatch warning: ERROR: "foo * bar" should be "foo *bar". Signed-off-by: Viresh Kumar --- include/linux/pm_opp.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index 25e47ab937b9..c6c7d73eb015 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -148,7 +148,7 @@ struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name) void dev_pm_opp_put_prop_name(struct opp_table *opp_table); struct opp_table *dev_pm_opp_set_regulators(struct device *dev, const char * const names[], unsigned int count); void dev_pm_opp_put_regulators(struct opp_table *opp_table); -struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char * name); +struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char *name); void dev_pm_opp_put_clkname(struct opp_table *opp_table); struct opp_table *dev_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)); void dev_pm_opp_unregister_set_opp_helper(struct opp_table *opp_table); @@ -347,7 +347,7 @@ static inline struct opp_table *dev_pm_opp_set_regulators(struct device *dev, co static inline void dev_pm_opp_put_regulators(struct opp_table *opp_table) {} -static inline struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char * name) +static inline struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char *name) { return ERR_PTR(-ENOTSUPP); } -- cgit v1.2.3 From 1d614920318b914f86c1fec2adec06ad2f7c3f55 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 1 Feb 2021 10:48:54 +0530 Subject: opp: Replace ENOTSUPP with EOPNOTSUPP Checkpatch gives following warning for new patches, and the new patches normally follow the existing standards for such stuff. Lets fix it properly. WARNING: ENOTSUPP is not a SUSV4 error code, prefer EOPNOTSUPP. Signed-off-by: Viresh Kumar --- include/linux/pm_opp.h | 64 +++++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index c6c7d73eb015..ab1d15ce559d 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -167,12 +167,12 @@ int dev_pm_opp_sync_regulators(struct device *dev); #else static inline struct opp_table *dev_pm_opp_get_opp_table(struct device *dev) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline struct opp_table *dev_pm_opp_get_opp_table_indexed(struct device *dev, int index) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline void dev_pm_opp_put_opp_table(struct opp_table *opp_table) {} @@ -232,37 +232,37 @@ static inline unsigned long dev_pm_opp_get_suspend_opp_freq(struct device *dev) static inline struct dev_pm_opp *dev_pm_opp_find_freq_exact(struct device *dev, unsigned long freq, bool available) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline struct dev_pm_opp *dev_pm_opp_find_level_exact(struct device *dev, unsigned int level) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline struct dev_pm_opp *dev_pm_opp_find_level_ceil(struct device *dev, unsigned int *level) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev, unsigned long *freq) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline struct dev_pm_opp *dev_pm_opp_find_freq_ceil_by_volt(struct device *dev, unsigned long u_volt) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline struct dev_pm_opp *dev_pm_opp_find_freq_ceil(struct device *dev, unsigned long *freq) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline void dev_pm_opp_put(struct dev_pm_opp *opp) {} @@ -270,7 +270,7 @@ static inline void dev_pm_opp_put(struct dev_pm_opp *opp) {} static inline int dev_pm_opp_add(struct device *dev, unsigned long freq, unsigned long u_volt) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline void dev_pm_opp_remove(struct device *dev, unsigned long freq) @@ -301,19 +301,19 @@ static inline int dev_pm_opp_disable(struct device *dev, unsigned long freq) static inline int dev_pm_opp_register_notifier(struct device *dev, struct notifier_block *nb) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int dev_pm_opp_unregister_notifier(struct device *dev, struct notifier_block *nb) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline struct opp_table *dev_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, unsigned int count) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline void dev_pm_opp_put_supported_hw(struct opp_table *opp_table) {} @@ -321,7 +321,7 @@ static inline void dev_pm_opp_put_supported_hw(struct opp_table *opp_table) {} static inline struct opp_table *dev_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline void dev_pm_opp_unregister_set_opp_helper(struct opp_table *opp_table) {} @@ -330,33 +330,33 @@ static inline struct opp_table * devm_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline struct opp_table *dev_pm_opp_set_prop_name(struct device *dev, const char *name) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline void dev_pm_opp_put_prop_name(struct opp_table *opp_table) {} static inline struct opp_table *dev_pm_opp_set_regulators(struct device *dev, const char * const names[], unsigned int count) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline void dev_pm_opp_put_regulators(struct opp_table *opp_table) {} static inline struct opp_table *dev_pm_opp_set_clkname(struct device *dev, const char *name) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline void dev_pm_opp_put_clkname(struct opp_table *opp_table) {} static inline struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, const char **names, struct device ***virt_devs) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline void dev_pm_opp_detach_genpd(struct opp_table *opp_table) {} @@ -364,27 +364,27 @@ static inline void dev_pm_opp_detach_genpd(struct opp_table *opp_table) {} static inline struct opp_table *devm_pm_opp_attach_genpd(struct device *dev, const char **names, struct device ***virt_devs) { - return ERR_PTR(-ENOTSUPP); + return ERR_PTR(-EOPNOTSUPP); } static inline int dev_pm_opp_xlate_performance_state(struct opp_table *src_table, struct opp_table *dst_table, unsigned int pstate) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int dev_pm_opp_set_opp(struct device *dev, struct dev_pm_opp *opp) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, const struct cpumask *cpumask) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int dev_pm_opp_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask) @@ -402,7 +402,7 @@ static inline void dev_pm_opp_cpumask_remove_table(const struct cpumask *cpumask static inline int dev_pm_opp_sync_regulators(struct device *dev) { - return -ENOTSUPP; + return -EOPNOTSUPP; } #endif /* CONFIG_PM_OPP */ @@ -427,17 +427,17 @@ static inline void dev_pm_opp_of_unregister_em(struct device *dev) #else static inline int dev_pm_opp_of_add_table(struct device *dev) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int dev_pm_opp_of_add_table_indexed(struct device *dev, int index) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int dev_pm_opp_of_add_table_noclk(struct device *dev, int index) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline void dev_pm_opp_of_remove_table(struct device *dev) @@ -446,7 +446,7 @@ static inline void dev_pm_opp_of_remove_table(struct device *dev) static inline int dev_pm_opp_of_cpumask_add_table(const struct cpumask *cpumask) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline void dev_pm_opp_of_cpumask_remove_table(const struct cpumask *cpumask) @@ -455,7 +455,7 @@ static inline void dev_pm_opp_of_cpumask_remove_table(const struct cpumask *cpum static inline int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline struct device_node *dev_pm_opp_of_get_opp_desc_node(struct device *dev) @@ -471,7 +471,7 @@ static inline struct device_node *dev_pm_opp_get_of_node(struct dev_pm_opp *opp) static inline int dev_pm_opp_of_register_em(struct device *dev, struct cpumask *cpus) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline void dev_pm_opp_of_unregister_em(struct device *dev) @@ -480,12 +480,12 @@ static inline void dev_pm_opp_of_unregister_em(struct device *dev) static inline int of_get_required_opp_performance_state(struct device_node *np, int index) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int dev_pm_opp_of_find_icc_paths(struct device *dev, struct opp_table *opp_table) { - return -ENOTSUPP; + return -EOPNOTSUPP; } #endif -- cgit v1.2.3 From 3254899e0b52f10b9a3e7db4d10f081f60705ba9 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Thu, 21 Jan 2021 09:09:47 +0000 Subject: nvme: update enumerations for status codes All the updates are mentioned in the ratified NVMe 1.4 spec. Reviewed-by: Hannes Reinecke Signed-off-by: Max Gurtovoy Signed-off-by: Christoph Hellwig --- include/linux/nvme.h | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/nvme.h b/include/linux/nvme.h index bfed36e342cc..458719544253 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -1473,20 +1473,29 @@ enum { NVME_SC_SGL_INVALID_DATA = 0xf, NVME_SC_SGL_INVALID_METADATA = 0x10, NVME_SC_SGL_INVALID_TYPE = 0x11, - + NVME_SC_CMB_INVALID_USE = 0x12, + NVME_SC_PRP_INVALID_OFFSET = 0x13, + NVME_SC_ATOMIC_WU_EXCEEDED = 0x14, + NVME_SC_OP_DENIED = 0x15, NVME_SC_SGL_INVALID_OFFSET = 0x16, - NVME_SC_SGL_INVALID_SUBTYPE = 0x17, - + NVME_SC_RESERVED = 0x17, + NVME_SC_HOST_ID_INCONSIST = 0x18, + NVME_SC_KA_TIMEOUT_EXPIRED = 0x19, + NVME_SC_KA_TIMEOUT_INVALID = 0x1A, + NVME_SC_ABORTED_PREEMPT_ABORT = 0x1B, NVME_SC_SANITIZE_FAILED = 0x1C, NVME_SC_SANITIZE_IN_PROGRESS = 0x1D, - + NVME_SC_SGL_INVALID_GRANULARITY = 0x1E, + NVME_SC_CMD_NOT_SUP_CMB_QUEUE = 0x1F, NVME_SC_NS_WRITE_PROTECTED = 0x20, NVME_SC_CMD_INTERRUPTED = 0x21, + NVME_SC_TRANSIENT_TR_ERR = 0x22, NVME_SC_LBA_RANGE = 0x80, NVME_SC_CAP_EXCEEDED = 0x81, NVME_SC_NS_NOT_READY = 0x82, NVME_SC_RESERVATION_CONFLICT = 0x83, + NVME_SC_FORMAT_IN_PROGRESS = 0x84, /* * Command Specific Status: @@ -1519,8 +1528,15 @@ enum { NVME_SC_NS_NOT_ATTACHED = 0x11a, NVME_SC_THIN_PROV_NOT_SUPP = 0x11b, NVME_SC_CTRL_LIST_INVALID = 0x11c, + NVME_SC_SELT_TEST_IN_PROGRESS = 0x11d, NVME_SC_BP_WRITE_PROHIBITED = 0x11e, + NVME_SC_CTRL_ID_INVALID = 0x11f, + NVME_SC_SEC_CTRL_STATE_INVALID = 0x120, + NVME_SC_CTRL_RES_NUM_INVALID = 0x121, + NVME_SC_RES_ID_INVALID = 0x122, NVME_SC_PMR_SAN_PROHIBITED = 0x123, + NVME_SC_ANA_GROUP_ID_INVALID = 0x124, + NVME_SC_ANA_ATTACH_FAILED = 0x125, /* * I/O Command Set Specific - NVM commands: -- cgit v1.2.3 From 4a407d5ebc7ac1ea8c6e2692bd79320459dc60f6 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Wed, 27 Jan 2021 02:50:00 +0900 Subject: nvme: add tracing of zns commands When support for the NVMe ZNS commands was merged, tracing of these has been omitted. Add nvme_cmd_zone_mgmt_send, nvme_cmd_zone_mgmt_recv as well as nvme_cmd_zone_append to the nvme driver's tracing facility. Signed-off-by: Johannes Thumshirn Signed-off-by: Christoph Hellwig --- drivers/nvme/host/trace.c | 34 ++++++++++++++++++++++++++++++++++ include/linux/nvme.h | 6 +++++- 2 files changed, 39 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/nvme/host/trace.c b/drivers/nvme/host/trace.c index e0400de713b5..6543015b6121 100644 --- a/drivers/nvme/host/trace.c +++ b/drivers/nvme/host/trace.c @@ -148,6 +148,35 @@ static const char *nvme_trace_dsm(struct trace_seq *p, u8 *cdw10) return ret; } +static const char *nvme_trace_zone_mgmt_send(struct trace_seq *p, u8 *cdw10) +{ + const char *ret = trace_seq_buffer_ptr(p); + u64 slba = get_unaligned_le64(cdw10); + u8 zsa = cdw10[12]; + u8 all = cdw10[13]; + + trace_seq_printf(p, "slba=%llu, zsa=%u, all=%u", slba, zsa, all); + trace_seq_putc(p, 0); + + return ret; +} + +static const char *nvme_trace_zone_mgmt_recv(struct trace_seq *p, u8 *cdw10) +{ + const char *ret = trace_seq_buffer_ptr(p); + u64 slba = get_unaligned_le64(cdw10); + u32 numd = get_unaligned_le32(cdw10 + 8); + u8 zra = cdw10[12]; + u8 zrasf = cdw10[13]; + u8 pr = cdw10[14]; + + trace_seq_printf(p, "slba=%llu, numd=%u, zra=%u, zrasf=%u, pr=%u", + slba, numd, zra, zrasf, pr); + trace_seq_putc(p, 0); + + return ret; +} + static const char *nvme_trace_common(struct trace_seq *p, u8 *cdw10) { const char *ret = trace_seq_buffer_ptr(p); @@ -190,9 +219,14 @@ const char *nvme_trace_parse_nvm_cmd(struct trace_seq *p, case nvme_cmd_read: case nvme_cmd_write: case nvme_cmd_write_zeroes: + case nvme_cmd_zone_append: return nvme_trace_read_write(p, cdw10); case nvme_cmd_dsm: return nvme_trace_dsm(p, cdw10); + case nvme_cmd_zone_mgmt_send: + return nvme_trace_zone_mgmt_send(p, cdw10); + case nvme_cmd_zone_mgmt_recv: + return nvme_trace_zone_mgmt_recv(p, cdw10); default: return nvme_trace_common(p, cdw10); } diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 458719544253..b08787cd0881 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -697,7 +697,11 @@ enum nvme_opcode { nvme_opcode_name(nvme_cmd_resv_register), \ nvme_opcode_name(nvme_cmd_resv_report), \ nvme_opcode_name(nvme_cmd_resv_acquire), \ - nvme_opcode_name(nvme_cmd_resv_release)) + nvme_opcode_name(nvme_cmd_resv_release), \ + nvme_opcode_name(nvme_cmd_zone_mgmt_send), \ + nvme_opcode_name(nvme_cmd_zone_mgmt_recv), \ + nvme_opcode_name(nvme_cmd_zone_append)) + /* -- cgit v1.2.3 From c7020068bf2397b60bb62a1f71ca0fe626c1f7e7 Mon Sep 17 00:00:00 2001 From: Elvira Khabirova Date: Sun, 20 Sep 2020 04:58:57 +0300 Subject: tee: fix some comment typos in header files struct tee_param: revc -> recv. TEE_IOC_SUPPL_SEND: typo introduced by copy-pasting, replace invalid description with description from the according argument struct. Signed-off-by: Elvira Khabirova Signed-off-by: Jens Wiklander --- include/linux/tee_drv.h | 2 +- include/uapi/linux/tee.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tee_drv.h b/include/linux/tee_drv.h index cdd049a724b1..54269e47ac9a 100644 --- a/include/linux/tee_drv.h +++ b/include/linux/tee_drv.h @@ -88,7 +88,7 @@ struct tee_param { * @close_session: close a session * @invoke_func: invoke a trusted function * @cancel_req: request cancel of an ongoing invoke or open - * @supp_revc: called for supplicant to get a command + * @supp_recv: called for supplicant to get a command * @supp_send: called for supplicant to send a response * @shm_register: register shared memory buffer in TEE * @shm_unregister: unregister shared memory buffer in TEE diff --git a/include/uapi/linux/tee.h b/include/uapi/linux/tee.h index d67cadf221fc..25a6c534beb1 100644 --- a/include/uapi/linux/tee.h +++ b/include/uapi/linux/tee.h @@ -355,7 +355,7 @@ struct tee_iocl_supp_send_arg { }; /** - * TEE_IOC_SUPPL_SEND - Receive a request for a supplicant function + * TEE_IOC_SUPPL_SEND - Send a response to a received request * * Takes a struct tee_ioctl_buf_data which contains a struct * tee_iocl_supp_send_arg followed by any array of struct tee_param -- cgit v1.2.3 From c8b186a8d54d7e12d28e9f9686cb00ff18fc2ab2 Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Tue, 2 Feb 2021 18:23:26 +1100 Subject: tracepoint: Fix race between tracing and removing tracepoint When executing a tracepoint, the tracepoint's func is dereferenced twice - in __DO_TRACE() (where the returned pointer is checked) and later on in __traceiter_##_name where the returned pointer is dereferenced without checking which leads to races against tracepoint_removal_sync() and crashes. This adds a check before referencing the pointer in tracepoint_ptr_deref. Link: https://lkml.kernel.org/r/20210202072326.120557-1-aik@ozlabs.ru Cc: stable@vger.kernel.org Fixes: d25e37d89dd2f ("tracepoint: Optimize using static_call()") Acked-by: Peter Zijlstra (Intel) Signed-off-by: Alexey Kardashevskiy Signed-off-by: Steven Rostedt (VMware) --- include/linux/tracepoint.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h index 0f21617f1a66..966ed8980327 100644 --- a/include/linux/tracepoint.h +++ b/include/linux/tracepoint.h @@ -307,11 +307,13 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) \ it_func_ptr = \ rcu_dereference_raw((&__tracepoint_##_name)->funcs); \ - do { \ - it_func = (it_func_ptr)->func; \ - __data = (it_func_ptr)->data; \ - ((void(*)(void *, proto))(it_func))(__data, args); \ - } while ((++it_func_ptr)->func); \ + if (it_func_ptr) { \ + do { \ + it_func = (it_func_ptr)->func; \ + __data = (it_func_ptr)->data; \ + ((void(*)(void *, proto))(it_func))(__data, args); \ + } while ((++it_func_ptr)->func); \ + } \ return 0; \ } \ DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name); -- cgit v1.2.3 From 4c9fb5d9140802db4db9f66c23887f43174e113c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 2 Feb 2021 15:54:19 +0100 Subject: iommu: Check dev->iommu in dev_iommu_priv_get() before dereferencing it The dev_iommu_priv_get() needs a similar check to dev_iommu_fwspec_get() to make sure no NULL-ptr is dereferenced. Fixes: 05a0542b456e1 ("iommu/amd: Store dev_data as device iommu private data") Cc: stable@vger.kernel.org # v5.8+ Link: https://lore.kernel.org/r/20210202145419.29143-1-joro@8bytes.org Reference: https://bugzilla.kernel.org/show_bug.cgi?id=211241 Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/iommu.h b/include/linux/iommu.h index b3f0e2018c62..efa96263b81b 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -616,7 +616,10 @@ static inline void dev_iommu_fwspec_set(struct device *dev, static inline void *dev_iommu_priv_get(struct device *dev) { - return dev->iommu->priv; + if (dev->iommu) + return dev->iommu->priv; + else + return NULL; } static inline void dev_iommu_priv_set(struct device *dev, void *priv) -- cgit v1.2.3 From a69bdb283f79949b67632878ef1822badae9299f Mon Sep 17 00:00:00 2001 From: Badhri Jagan Sridharan Date: Mon, 1 Feb 2021 16:30:59 -0800 Subject: usb: typec: tcpm: Add Callback to Usb Communication capable partner The USB Communications Capable bit indicates if port partner is capable of communication over the USB data lines (e.g. D+/- or SS Tx/Rx). Notify the status of the bit to low level drivers to perform chip specific operation. For instance, low level driver enables USB switches on D+/D- lines to set up data path when the bit is set. Refactored from patch initially authored by Kyle Tso Reviewed-by: Heikki Krogerus Reviewed-by: Guenter Roeck Signed-off-by: Badhri Jagan Sridharan Link: https://lore.kernel.org/r/20210202003101.221145-1-badhri@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tcpm/tcpm.c | 13 +++++++++++++ include/linux/usb/tcpm.h | 5 +++++ 2 files changed, 18 insertions(+) (limited to 'include/linux') diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 7747c7a154a4..8558ab006885 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -3430,6 +3430,14 @@ static void tcpm_unregister_altmodes(struct tcpm_port *port) memset(modep, 0, sizeof(*modep)); } +static void tcpm_set_partner_usb_comm_capable(struct tcpm_port *port, bool capable) +{ + tcpm_log(port, "Setting usb_comm capable %s", capable ? "true" : "false"); + + if (port->tcpc->set_partner_usb_comm_capable) + port->tcpc->set_partner_usb_comm_capable(port->tcpc, capable); +} + static void tcpm_reset_port(struct tcpm_port *port) { int ret; @@ -3446,6 +3454,7 @@ static void tcpm_reset_port(struct tcpm_port *port) port->attached = false; port->pd_capable = false; port->pps_data.supported = false; + tcpm_set_partner_usb_comm_capable(port, false); /* * First Rx ID should be 0; set this to a sentinel of -1 so that @@ -3786,6 +3795,8 @@ static void run_state_machine(struct tcpm_port *port) } } else { tcpm_pd_send_control(port, PD_CTRL_ACCEPT); + tcpm_set_partner_usb_comm_capable(port, + !!(port->sink_request & RDO_USB_COMM)); tcpm_set_state(port, SRC_TRANSITION_SUPPLY, PD_T_SRC_TRANSITION); } @@ -4005,6 +4016,8 @@ static void run_state_machine(struct tcpm_port *port) break; case SNK_NEGOTIATE_CAPABILITIES: port->pd_capable = true; + tcpm_set_partner_usb_comm_capable(port, + !!(port->source_caps[0] & PDO_FIXED_USB_COMM)); port->hard_reset_count = 0; ret = tcpm_pd_send_request(port); if (ret < 0) { diff --git a/include/linux/usb/tcpm.h b/include/linux/usb/tcpm.h index 3af99f85e8b9..42fcfbe10590 100644 --- a/include/linux/usb/tcpm.h +++ b/include/linux/usb/tcpm.h @@ -108,6 +108,10 @@ enum tcpm_transmit_type { * is supported by TCPC, set this callback for TCPM to query * whether vbus is at VSAFE0V when needed. * Returns true when vbus is at VSAFE0V, false otherwise. + * @set_partner_usb_comm_capable: + * Optional; The USB Communications Capable bit indicates if port + * partner is capable of communication over the USB data lines + * (e.g. D+/- or SS Tx/Rx). Called to notify the status of the bit. */ struct tcpc_dev { struct fwnode_handle *fwnode; @@ -139,6 +143,7 @@ struct tcpc_dev { int (*set_auto_vbus_discharge_threshold)(struct tcpc_dev *dev, enum typec_pwr_opmode mode, bool pps_active, u32 requested_vbus_voltage); bool (*is_vbus_vsafe0v)(struct tcpc_dev *dev); + void (*set_partner_usb_comm_capable)(struct tcpc_dev *dev, bool enable); }; struct tcpm_port; -- cgit v1.2.3 From 32e9b48d110ef5fae850036eafaf7895a25b37e3 Mon Sep 17 00:00:00 2001 From: Kyle Tso Date: Tue, 2 Feb 2021 17:55:12 +0800 Subject: usb: typec: Return void in typec_partner_set_pd_revision typec_partner_set_pd_revision doesn't need any return value. Fixes: 29b01295a829 ("usb: typec: Add typec_partner_set_pd_revision") Reviewed-by: Heikki Krogerus Reviewed-by: Benson Leung Signed-off-by: Kyle Tso Link: https://lore.kernel.org/r/20210202095512.761214-1-kyletso@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/class.c | 10 ++-------- include/linux/usb/typec.h | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index b6ceab3dc16b..a7d1bc83c2d4 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -755,15 +755,11 @@ EXPORT_SYMBOL_GPL(typec_partner_set_identity); * * This routine is used to report that the PD revision of the port partner has * become available. - * - * Returns 0 on success or negative error number on failure. */ -int typec_partner_set_pd_revision(struct typec_partner *partner, u16 pd_revision) +void typec_partner_set_pd_revision(struct typec_partner *partner, u16 pd_revision) { - int ret; - if (partner->pd_revision == pd_revision) - return 0; + return; partner->pd_revision = pd_revision; sysfs_notify(&partner->dev.kobj, NULL, "usb_power_delivery_revision"); @@ -773,8 +769,6 @@ int typec_partner_set_pd_revision(struct typec_partner *partner, u16 pd_revision "supports_usb_power_delivery"); } kobject_uevent(&partner->dev.kobj, KOBJ_CHANGE); - - return 0; } EXPORT_SYMBOL_GPL(typec_partner_set_pd_revision); diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h index 4946eca742d5..a94df82ab62f 100644 --- a/include/linux/usb/typec.h +++ b/include/linux/usb/typec.h @@ -126,7 +126,7 @@ struct typec_altmode_desc { enum typec_port_data roles; }; -int typec_partner_set_pd_revision(struct typec_partner *partner, u16 pd_revision); +void typec_partner_set_pd_revision(struct typec_partner *partner, u16 pd_revision); int typec_partner_set_num_altmodes(struct typec_partner *partner, int num_altmodes); struct typec_altmode *typec_partner_register_altmode(struct typec_partner *partner, -- cgit v1.2.3 From 0e1d6f55a12e47942ce207dfb93e23049b454c9e Mon Sep 17 00:00:00 2001 From: Kyle Tso Date: Wed, 3 Feb 2021 00:17:27 +0800 Subject: usb: pd: Update VDO definitions "PD Spec Revision 3.0 Version 2.0 + ECNs 2020-12-10" introduces several changes regarding the ID Header VDO and the Product Type VDOs. Signed-off-by: Kyle Tso Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20210202161733.932215-3-kyletso@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/class.c | 8 +- include/linux/usb/pd_vdo.h | 308 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 242 insertions(+), 74 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index a7d1bc83c2d4..b4a5d9d4564f 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -88,7 +88,7 @@ static const char * const typec_accessory_modes[] = { /* Product types defined in USB PD Specification R3.0 V2.0 */ static const char * const product_type_ufp[8] = { - [IDH_PTYPE_UNDEF] = "undefined", + [IDH_PTYPE_NOT_UFP] = "not_ufp", [IDH_PTYPE_HUB] = "hub", [IDH_PTYPE_PERIPH] = "peripheral", [IDH_PTYPE_PSD] = "psd", @@ -96,17 +96,17 @@ static const char * const product_type_ufp[8] = { }; static const char * const product_type_dfp[8] = { - [IDH_PTYPE_DFP_UNDEF] = "undefined", + [IDH_PTYPE_NOT_DFP] = "not_dfp", [IDH_PTYPE_DFP_HUB] = "hub", [IDH_PTYPE_DFP_HOST] = "host", [IDH_PTYPE_DFP_PB] = "power_brick", - [IDH_PTYPE_DFP_AMC] = "amc", }; static const char * const product_type_cable[8] = { - [IDH_PTYPE_UNDEF] = "undefined", + [IDH_PTYPE_NOT_CABLE] = "not_cable", [IDH_PTYPE_PCABLE] = "passive", [IDH_PTYPE_ACABLE] = "active", + [IDH_PTYPE_VPD] = "vpd", }; static struct usb_pd_identity *get_pd_identity(struct device *dev) diff --git a/include/linux/usb/pd_vdo.h b/include/linux/usb/pd_vdo.h index 8c08eeb9a74b..e9b6822c54c2 100644 --- a/include/linux/usb/pd_vdo.h +++ b/include/linux/usb/pd_vdo.h @@ -103,34 +103,46 @@ * -------------------- * <31> :: data capable as a USB host * <30> :: data capable as a USB device - * <29:27> :: product type (UFP / Cable) + * <29:27> :: product type (UFP / Cable / VPD) * <26> :: modal operation supported (1b == yes) - * <25:16> :: product type (DFP) + * <25:23> :: product type (DFP) (SVDM version 2.0+ only; set to zero in version 1.0) + * <22:21> :: connector type (SVDM version 2.0+ only; set to zero in version 1.0) + * <20:16> :: Reserved, Shall be set to zero * <15:0> :: USB-IF assigned VID for this cable vendor */ -#define IDH_PTYPE_UNDEF 0 +/* SOP Product Type (UFP) */ +#define IDH_PTYPE_NOT_UFP 0 #define IDH_PTYPE_HUB 1 #define IDH_PTYPE_PERIPH 2 #define IDH_PTYPE_PSD 3 #define IDH_PTYPE_AMA 5 +/* SOP' Product Type (Cable Plug / VPD) */ +#define IDH_PTYPE_NOT_CABLE 0 #define IDH_PTYPE_PCABLE 3 #define IDH_PTYPE_ACABLE 4 +#define IDH_PTYPE_VPD 6 -#define IDH_PTYPE_DFP_UNDEF 0 +/* SOP Product Type (DFP) */ +#define IDH_PTYPE_NOT_DFP 0 #define IDH_PTYPE_DFP_HUB 1 #define IDH_PTYPE_DFP_HOST 2 #define IDH_PTYPE_DFP_PB 3 -#define IDH_PTYPE_DFP_AMC 4 -#define VDO_IDH(usbh, usbd, ptype, is_modal, vid) \ - ((usbh) << 31 | (usbd) << 30 | ((ptype) & 0x7) << 27 \ - | (is_modal) << 26 | ((vid) & 0xffff)) +/* ID Header Mask */ +#define IDH_DFP_MASK GENMASK(25, 23) +#define IDH_CONN_MASK GENMASK(22, 21) + +#define VDO_IDH(usbh, usbd, ufp_cable, is_modal, dfp, conn, vid) \ + ((usbh) << 31 | (usbd) << 30 | ((ufp_cable) & 0x7) << 27 \ + | (is_modal) << 26 | ((dfp) & 0x7) << 23 | ((conn) & 0x3) << 21 \ + | ((vid) & 0xffff)) #define PD_IDH_PTYPE(vdo) (((vdo) >> 27) & 0x7) #define PD_IDH_VID(vdo) ((vdo) & 0xffff) #define PD_IDH_MODAL_SUPP(vdo) ((vdo) & (1 << 26)) #define PD_IDH_DFP_PTYPE(vdo) (((vdo) >> 23) & 0x7) +#define PD_IDH_CONN_TYPE(vdo) (((vdo) >> 21) & 0x3) /* * Cert Stat VDO @@ -138,6 +150,7 @@ * <31:0> : USB-IF assigned XID for this cable */ #define PD_CSTAT_XID(vdo) (vdo) +#define VDO_CERT(xid) ((xid) & 0xffffffff) /* * Product VDO @@ -149,112 +162,267 @@ #define PD_PRODUCT_PID(vdo) (((vdo) >> 16) & 0xffff) /* - * UFP VDO1 + * UFP VDO (PD Revision 3.0+ only) * -------- * <31:29> :: UFP VDO version * <28> :: Reserved * <27:24> :: Device capability - * <23:6> :: Reserved + * <23:22> :: Connector type (10b == receptacle, 11b == captive plug) + * <21:11> :: Reserved + * <10:8> :: Vconn power (AMA only) + * <7> :: Vconn required (AMA only, 0b == no, 1b == yes) + * <6> :: Vbus required (AMA only, 0b == yes, 1b == no) * <5:3> :: Alternate modes * <2:0> :: USB highest speed */ -#define PD_VDO1_UFP_DEVCAP(vdo) (((vdo) & GENMASK(27, 24)) >> 24) +#define PD_VDO_UFP_DEVCAP(vdo) (((vdo) & GENMASK(27, 24)) >> 24) + +/* UFP VDO Version */ +#define UFP_VDO_VER1_2 2 +/* Device Capability */ #define DEV_USB2_CAPABLE BIT(0) #define DEV_USB2_BILLBOARD BIT(1) #define DEV_USB3_CAPABLE BIT(2) #define DEV_USB4_CAPABLE BIT(3) +/* Connector Type */ +#define UFP_RECEPTACLE 2 +#define UFP_CAPTIVE 3 + +/* Vconn Power (AMA only, set to AMA_VCONN_NOT_REQ if Vconn is not required) */ +#define AMA_VCONN_PWR_1W 0 +#define AMA_VCONN_PWR_1W5 1 +#define AMA_VCONN_PWR_2W 2 +#define AMA_VCONN_PWR_3W 3 +#define AMA_VCONN_PWR_4W 4 +#define AMA_VCONN_PWR_5W 5 +#define AMA_VCONN_PWR_6W 6 + +/* Vconn Required (AMA only) */ +#define AMA_VCONN_NOT_REQ 0 +#define AMA_VCONN_REQ 1 + +/* Vbus Required (AMA only) */ +#define AMA_VBUS_REQ 0 +#define AMA_VBUS_NOT_REQ 1 + +/* Alternate Modes */ +#define UFP_ALTMODE_NOT_SUPP 0 +#define UFP_ALTMODE_TBT3 BIT(0) +#define UFP_ALTMODE_RECFG BIT(1) +#define UFP_ALTMODE_NO_RECFG BIT(2) + +/* USB Highest Speed */ +#define UFP_USB2_ONLY 0 +#define UFP_USB32_GEN1 1 +#define UFP_USB32_4_GEN2 2 +#define UFP_USB4_GEN3 3 + +#define VDO_UFP(ver, cap, conn, vcpwr, vcr, vbr, alt, spd) \ + (((ver) & 0x7) << 29 | ((cap) & 0xf) << 24 | ((conn) & 0x3) << 22 \ + | ((vcpwr) & 0x7) << 8 | (vcr) << 7 | (vbr) << 6 | ((alt) & 0x7) << 3 \ + | ((spd) & 0x7)) + /* - * DFP VDO + * DFP VDO (PD Revision 3.0+ only) * -------- * <31:29> :: DFP VDO version * <28:27> :: Reserved * <26:24> :: Host capability - * <23:5> :: Reserved + * <23:22> :: Connector type (10b == receptacle, 11b == captive plug) + * <21:5> :: Reserved * <4:0> :: Port number */ #define PD_VDO_DFP_HOSTCAP(vdo) (((vdo) & GENMASK(26, 24)) >> 24) +#define DFP_VDO_VER1_1 1 #define HOST_USB2_CAPABLE BIT(0) #define HOST_USB3_CAPABLE BIT(1) #define HOST_USB4_CAPABLE BIT(2) +#define DFP_RECEPTACLE 2 +#define DFP_CAPTIVE 3 + +#define VDO_DFP(ver, cap, conn, pnum) \ + (((ver) & 0x7) << 29 | ((cap) & 0x7) << 24 | ((conn) & 0x3) << 22 \ + | ((pnum) & 0x1f)) /* - * Cable VDO + * Passive Cable VDO * --------- * <31:28> :: Cable HW version * <27:24> :: Cable FW version - * <23:20> :: Reserved, Shall be set to zero - * <19:18> :: type-C to Type-A/B/C/Captive (00b == A, 01 == B, 10 == C, 11 == Captive) - * <17> :: Type-C to Plug/Receptacle (0b == plug, 1b == receptacle) + * <23:21> :: VDO version + * <20> :: Reserved, Shall be set to zero + * <19:18> :: Type-C to Type-C/Captive (10b == C, 11b == Captive) + * <17> :: Reserved, Shall be set to zero * <16:13> :: cable latency (0001 == <10ns(~1m length)) - * <12:11> :: cable termination type (11b == both ends active VCONN req) - * <10> :: SSTX1 Directionality support (0b == fixed, 1b == cfgable) - * <9> :: SSTX2 Directionality support - * <8> :: SSRX1 Directionality support - * <7> :: SSRX2 Directionality support - * <6:5> :: Vbus current handling capability + * <12:11> :: cable termination type (10b == Vconn not req, 01b == Vconn req) + * <10:9> :: Maximum Vbus voltage (00b == 20V, 01b == 30V, 10b == 40V, 11b == 50V) + * <8:7> :: Reserved, Shall be set to zero + * <6:5> :: Vbus current handling capability (01b == 3A, 10b == 5A) + * <4:3> :: Reserved, Shall be set to zero + * <2:0> :: USB highest speed + * + * Active Cable VDO 1 + * --------- + * <31:28> :: Cable HW version + * <27:24> :: Cable FW version + * <23:21> :: VDO version + * <20> :: Reserved, Shall be set to zero + * <19:18> :: Connector type (10b == C, 11b == Captive) + * <17> :: Reserved, Shall be set to zero + * <16:13> :: cable latency (0001 == <10ns(~1m length)) + * <12:11> :: cable termination type (10b == one end active, 11b == both ends active VCONN req) + * <10:9> :: Maximum Vbus voltage (00b == 20V, 01b == 30V, 10b == 40V, 11b == 50V) + * <8> :: SBU supported (0b == supported, 1b == not supported) + * <7> :: SBU type (0b == passive, 1b == active) + * <6:5> :: Vbus current handling capability (01b == 3A, 10b == 5A) * <4> :: Vbus through cable (0b == no, 1b == yes) * <3> :: SOP" controller present? (0b == no, 1b == yes) - * <2:0> :: USB SS Signaling support + * <2:0> :: USB highest speed */ -#define CABLE_ATYPE 0 -#define CABLE_BTYPE 1 +/* Cable VDO Version */ +#define CABLE_VDO_VER1_0 0 +#define CABLE_VDO_VER1_3 3 + +/* Connector Type */ #define CABLE_CTYPE 2 #define CABLE_CAPTIVE 3 -#define CABLE_PLUG 0 -#define CABLE_RECEPTACLE 1 -#define CABLE_CURR_1A5 0 + +/* Cable Latency */ +#define CABLE_LATENCY_1M 1 +#define CABLE_LATENCY_2M 2 +#define CABLE_LATENCY_3M 3 +#define CABLE_LATENCY_4M 4 +#define CABLE_LATENCY_5M 5 +#define CABLE_LATENCY_6M 6 +#define CABLE_LATENCY_7M 7 +#define CABLE_LATENCY_7M_PLUS 8 + +/* Cable Termination Type */ +#define PCABLE_VCONN_NOT_REQ 0 +#define PCABLE_VCONN_REQ 1 +#define ACABLE_ONE_END 2 +#define ACABLE_BOTH_END 3 + +/* Maximum Vbus Voltage */ +#define CABLE_MAX_VBUS_20V 0 +#define CABLE_MAX_VBUS_30V 1 +#define CABLE_MAX_VBUS_40V 2 +#define CABLE_MAX_VBUS_50V 3 + +/* Active Cable SBU Supported/Type */ +#define ACABLE_SBU_SUPP 0 +#define ACABLE_SBU_NOT_SUPP 1 +#define ACABLE_SBU_PASSIVE 0 +#define ACABLE_SBU_ACTIVE 1 + +/* Vbus Current Handling Capability */ +#define CABLE_CURR_DEF 0 #define CABLE_CURR_3A 1 #define CABLE_CURR_5A 2 -#define CABLE_USBSS_U2_ONLY 0 -#define CABLE_USBSS_U31_GEN1 1 -#define CABLE_USBSS_U31_GEN2 2 -#define VDO_CABLE(hw, fw, cbl, gdr, lat, term, tx1d, tx2d, rx1d, rx2d, cur,\ - vps, sopp, usbss) \ - (((hw) & 0x7) << 28 | ((fw) & 0x7) << 24 | ((cbl) & 0x3) << 18 \ - | (gdr) << 17 | ((lat) & 0x7) << 13 | ((term) & 0x3) << 11 \ - | (tx1d) << 10 | (tx2d) << 9 | (rx1d) << 8 | (rx2d) << 7 \ - | ((cur) & 0x3) << 5 | (vps) << 4 | (sopp) << 3 \ - | ((usbss) & 0x7)) + +/* USB Highest Speed */ +#define CABLE_USB2_ONLY 0 +#define CABLE_USB32_GEN1 1 +#define CABLE_USB32_4_GEN2 2 +#define CABLE_USB4_GEN3 3 + +#define VDO_PCABLE(hw, fw, ver, conn, lat, term, vbm, cur, spd) \ + (((hw) & 0xf) << 28 | ((fw) & 0xf) << 24 | ((ver) & 0x7) << 21 \ + | ((conn) & 0x3) << 18 | ((lat) & 0xf) << 13 | ((term) & 0x3) << 11 \ + | ((vbm) & 0x3) << 9 | ((cur) & 0x3) << 5 | ((spd) & 0x7)) +#define VDO_ACABLE1(hw, fw, ver, conn, lat, term, vbm, sbu, sbut, cur, vbt, sopp, spd) \ + (((hw) & 0xf) << 28 | ((fw) & 0xf) << 24 | ((ver) & 0x7) << 21 \ + | ((conn) & 0x3) << 18 | ((lat) & 0xf) << 13 | ((term) & 0x3) << 11 \ + | ((vbm) & 0x3) << 9 | (sbu) << 8 | (sbut) << 7 | ((cur) & 0x3) << 5 \ + | (vbt) << 4 | (sopp) << 3 | ((spd) & 0x7)) + #define VDO_TYPEC_CABLE_TYPE(vdo) (((vdo) >> 18) & 0x3) /* - * AMA VDO + * Active Cable VDO 2 * --------- - * <31:28> :: Cable HW version - * <27:24> :: Cable FW version - * <23:12> :: Reserved, Shall be set to zero - * <11> :: SSTX1 Directionality support (0b == fixed, 1b == cfgable) - * <10> :: SSTX2 Directionality support - * <9> :: SSRX1 Directionality support - * <8> :: SSRX2 Directionality support - * <7:5> :: Vconn power - * <4> :: Vconn power required - * <3> :: Vbus power required - * <2:0> :: USB SS Signaling support + * <31:24> :: Maximum operating temperature + * <23:16> :: Shutdown temperature + * <15> :: Reserved, Shall be set to zero + * <14:12> :: U3/CLd power + * <11> :: U3 to U0 transition mode (0b == direct, 1b == through U3S) + * <10> :: Physical connection (0b == copper, 1b == optical) + * <9> :: Active element (0b == redriver, 1b == retimer) + * <8> :: USB4 supported (0b == yes, 1b == no) + * <7:6> :: USB2 hub hops consumed + * <5> :: USB2 supported (0b == yes, 1b == no) + * <4> :: USB3.2 supported (0b == yes, 1b == no) + * <3> :: USB lanes supported (0b == one lane, 1b == two lanes) + * <2> :: Optically isolated active cable (0b == no, 1b == yes) + * <1> :: Reserved, Shall be set to zero + * <0> :: USB gen (0b == gen1, 1b == gen2+) */ -#define VDO_AMA(hw, fw, tx1d, tx2d, rx1d, rx2d, vcpwr, vcr, vbr, usbss) \ - (((hw) & 0x7) << 28 | ((fw) & 0x7) << 24 \ - | (tx1d) << 11 | (tx2d) << 10 | (rx1d) << 9 | (rx2d) << 8 \ - | ((vcpwr) & 0x7) << 5 | (vcr) << 4 | (vbr) << 3 \ - | ((usbss) & 0x7)) - -#define PD_VDO_AMA_VCONN_REQ(vdo) (((vdo) >> 4) & 1) -#define PD_VDO_AMA_VBUS_REQ(vdo) (((vdo) >> 3) & 1) +/* U3/CLd Power*/ +#define ACAB2_U3_CLD_10MW_PLUS 0 +#define ACAB2_U3_CLD_10MW 1 +#define ACAB2_U3_CLD_5MW 2 +#define ACAB2_U3_CLD_1MW 3 +#define ACAB2_U3_CLD_500UW 4 +#define ACAB2_U3_CLD_200UW 5 +#define ACAB2_U3_CLD_50UW 6 + +/* Other Active Cable VDO 2 Fields */ +#define ACAB2_U3U0_DIRECT 0 +#define ACAB2_U3U0_U3S 1 +#define ACAB2_PHY_COPPER 0 +#define ACAB2_PHY_OPTICAL 1 +#define ACAB2_REDRIVER 0 +#define ACAB2_RETIMER 1 +#define ACAB2_USB4_SUPP 0 +#define ACAB2_USB4_NOT_SUPP 1 +#define ACAB2_USB2_SUPP 0 +#define ACAB2_USB2_NOT_SUPP 1 +#define ACAB2_USB32_SUPP 0 +#define ACAB2_USB32_NOT_SUPP 1 +#define ACAB2_LANES_ONE 0 +#define ACAB2_LANES_TWO 1 +#define ACAB2_OPT_ISO_NO 0 +#define ACAB2_OPT_ISO_YES 1 +#define ACAB2_GEN_1 0 +#define ACAB2_GEN_2_PLUS 1 + +#define VDO_ACABLE2(mtemp, stemp, u3p, trans, phy, ele, u4, hops, u2, u32, lane, iso, gen) \ + (((mtemp) & 0xff) << 24 | ((stemp) & 0xff) << 16 | ((u3p) & 0x7) << 12 \ + | (trans) << 11 | (phy) << 10 | (ele) << 9 | (u4) << 8 \ + | ((hops) & 0x3) << 6 | (u2) << 5 | (u32) << 4 | (lane) << 3 \ + | (iso) << 2 | (gen)) -#define AMA_VCONN_PWR_1W 0 -#define AMA_VCONN_PWR_1W5 1 -#define AMA_VCONN_PWR_2W 2 -#define AMA_VCONN_PWR_3W 3 -#define AMA_VCONN_PWR_4W 4 -#define AMA_VCONN_PWR_5W 5 -#define AMA_VCONN_PWR_6W 6 -#define AMA_USBSS_U2_ONLY 0 -#define AMA_USBSS_U31_GEN1 1 -#define AMA_USBSS_U31_GEN2 2 -#define AMA_USBSS_BBONLY 3 +/* + * VPD VDO + * --------- + * <31:28> :: HW version + * <27:24> :: FW version + * <23:21> :: VDO version + * <20:17> :: Reserved, Shall be set to zero + * <16:15> :: Maximum Vbus voltage (00b == 20V, 01b == 30V, 10b == 40V, 11b == 50V) + * <14> :: Charge through current support (0b == 3A, 1b == 5A) + * <13> :: Reserved, Shall be set to zero + * <12:7> :: Vbus impedance + * <6:1> :: Ground impedance + * <0> :: Charge through support (0b == no, 1b == yes) + */ +#define VPD_VDO_VER1_0 0 +#define VPD_MAX_VBUS_20V 0 +#define VPD_MAX_VBUS_30V 1 +#define VPD_MAX_VBUS_40V 2 +#define VPD_MAX_VBUS_50V 3 +#define VPDCT_CURR_3A 0 +#define VPDCT_CURR_5A 1 +#define VPDCT_NOT_SUPP 0 +#define VPDCT_SUPP 1 + +#define VDO_VPD(hw, fw, ver, vbm, curr, vbi, gi, ct) \ + (((hw) & 0xf) << 28 | ((fw) & 0xf) << 24 | ((ver) & 0x7) << 21 \ + | ((vbm) & 0x3) << 15 | (curr) << 14 | ((vbi) & 0x3f) << 7 \ + | ((gi) & 0x3f) << 1 | (ct)) /* * SVDM Discover SVIDs request -> response -- cgit v1.2.3 From 4f4e54366eae20d5867864001db57c5d90693d8c Mon Sep 17 00:00:00 2001 From: Emil Renner Berthing Date: Sun, 31 Jan 2021 00:46:37 +0100 Subject: net: usb: cdc_ncm: use new API for bh tasklet This converts the driver to use the new tasklet API introduced in commit 12cc923f1ccc ("tasklet: Introduce new initialization API") It is unfortunate that we need to add a pointer to the driver context to get back to the usbnet device, but the space will be reclaimed once there are no more users of the old API left and we can remove the data value and flag from the tasklet struct. Signed-off-by: Emil Renner Berthing Link: https://lore.kernel.org/r/20210130234637.26505-1-kernel@esmil.dk Signed-off-by: Jakub Kicinski --- drivers/net/usb/cdc_ncm.c | 12 +++++++----- include/linux/usb/cdc_ncm.h | 2 ++ 2 files changed, 9 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/usb/cdc_ncm.c b/drivers/net/usb/cdc_ncm.c index 291e76d32abe..4087c9e33781 100644 --- a/drivers/net/usb/cdc_ncm.c +++ b/drivers/net/usb/cdc_ncm.c @@ -61,7 +61,7 @@ static bool prefer_mbim; module_param(prefer_mbim, bool, 0644); MODULE_PARM_DESC(prefer_mbim, "Prefer MBIM setting on dual NCM/MBIM functions"); -static void cdc_ncm_txpath_bh(unsigned long param); +static void cdc_ncm_txpath_bh(struct tasklet_struct *t); static void cdc_ncm_tx_timeout_start(struct cdc_ncm_ctx *ctx); static enum hrtimer_restart cdc_ncm_tx_timer_cb(struct hrtimer *hr_timer); static struct usb_driver cdc_ncm_driver; @@ -813,9 +813,11 @@ int cdc_ncm_bind_common(struct usbnet *dev, struct usb_interface *intf, u8 data_ if (!ctx) return -ENOMEM; + ctx->dev = dev; + hrtimer_init(&ctx->tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ctx->tx_timer.function = &cdc_ncm_tx_timer_cb; - tasklet_init(&ctx->bh, cdc_ncm_txpath_bh, (unsigned long)dev); + tasklet_setup(&ctx->bh, cdc_ncm_txpath_bh); atomic_set(&ctx->stop, 0); spin_lock_init(&ctx->mtx); @@ -1472,10 +1474,10 @@ static enum hrtimer_restart cdc_ncm_tx_timer_cb(struct hrtimer *timer) return HRTIMER_NORESTART; } -static void cdc_ncm_txpath_bh(unsigned long param) +static void cdc_ncm_txpath_bh(struct tasklet_struct *t) { - struct usbnet *dev = (struct usbnet *)param; - struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0]; + struct cdc_ncm_ctx *ctx = from_tasklet(ctx, t, bh); + struct usbnet *dev = ctx->dev; spin_lock_bh(&ctx->mtx); if (ctx->tx_timer_pending != 0) { diff --git a/include/linux/usb/cdc_ncm.h b/include/linux/usb/cdc_ncm.h index 0ce4377545f8..f7cb3ddce7fb 100644 --- a/include/linux/usb/cdc_ncm.h +++ b/include/linux/usb/cdc_ncm.h @@ -98,6 +98,8 @@ struct cdc_ncm_ctx { struct hrtimer tx_timer; struct tasklet_struct bh; + struct usbnet *dev; + const struct usb_cdc_ncm_desc *func_desc; const struct usb_cdc_mbim_desc *mbim_desc; const struct usb_cdc_mbim_extended_desc *mbim_extended_desc; -- cgit v1.2.3 From 012ce4dd3102a0f4d80167de343e9d44b257c1b8 Mon Sep 17 00:00:00 2001 From: Danielle Ratson Date: Tue, 2 Feb 2021 20:06:06 +0200 Subject: ethtool: Extend link modes settings uAPI with lanes Currently, when auto negotiation is on, the user can advertise all the linkmodes which correspond to a specific speed, but does not have a similar selector for the number of lanes. This is significant when a specific speed can be achieved using different number of lanes. For example, 2x50 or 4x25. Add 'ETHTOOL_A_LINKMODES_LANES' attribute and expand 'struct ethtool_link_settings' with lanes field in order to implement a new lanes-selector that will enable the user to advertise a specific number of lanes as well. When auto negotiation is off, lanes parameter can be forced only if the driver supports it. Add a capability bit in 'struct ethtool_ops' that allows ethtool know if the driver can handle the lanes parameter when auto negotiation is off, so if it does not, an error message will be returned when trying to set lanes. Example: $ ethtool -s swp1 lanes 4 $ ethtool swp1 Settings for swp1: Supported ports: [ FIBRE ] Supported link modes: 1000baseKX/Full 10000baseKR/Full 40000baseCR4/Full 40000baseSR4/Full 40000baseLR4/Full 25000baseCR/Full 25000baseSR/Full 50000baseCR2/Full 100000baseSR4/Full 100000baseCR4/Full Supported pause frame use: Symmetric Receive-only Supports auto-negotiation: Yes Supported FEC modes: Not reported Advertised link modes: 40000baseCR4/Full 40000baseSR4/Full 40000baseLR4/Full 100000baseSR4/Full 100000baseCR4/Full Advertised pause frame use: No Advertised auto-negotiation: Yes Advertised FEC modes: Not reported Speed: Unknown! Duplex: Unknown! (255) Auto-negotiation: on Port: Direct Attach Copper PHYAD: 0 Transceiver: internal Link detected: no Signed-off-by: Danielle Ratson Signed-off-by: Jakub Kicinski --- Documentation/networking/ethtool-netlink.rst | 11 ++-- include/linux/ethtool.h | 4 ++ include/uapi/linux/ethtool_netlink.h | 1 + net/ethtool/linkmodes.c | 96 +++++++++++++++++++++++----- net/ethtool/netlink.h | 2 +- 5 files changed, 93 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/ethtool-netlink.rst b/Documentation/networking/ethtool-netlink.rst index 30b98245979f..05073482db05 100644 --- a/Documentation/networking/ethtool-netlink.rst +++ b/Documentation/networking/ethtool-netlink.rst @@ -431,16 +431,17 @@ Request contents: ``ETHTOOL_A_LINKMODES_SPEED`` u32 link speed (Mb/s) ``ETHTOOL_A_LINKMODES_DUPLEX`` u8 duplex mode ``ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG`` u8 Master/slave port mode + ``ETHTOOL_A_LINKMODES_LANES`` u32 lanes ========================================== ====== ========================== ``ETHTOOL_A_LINKMODES_OURS`` bit set allows setting advertised link modes. If autonegotiation is on (either set now or kept from before), advertised modes are not changed (no ``ETHTOOL_A_LINKMODES_OURS`` attribute) and at least one -of speed and duplex is specified, kernel adjusts advertised modes to all -supported modes matching speed, duplex or both (whatever is specified). This -autoselection is done on ethtool side with ioctl interface, netlink interface -is supposed to allow requesting changes without knowing what exactly kernel -supports. +of speed, duplex and lanes is specified, kernel adjusts advertised modes to all +supported modes matching speed, duplex, lanes or all (whatever is specified). +This autoselection is done on ethtool side with ioctl interface, netlink +interface is supposed to allow requesting changes without knowing what exactly +kernel supports. LINKSTATE_GET diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index e3da25b51ae4..1ab13c5dfb2f 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -128,6 +128,7 @@ struct ethtool_link_ksettings { __ETHTOOL_DECLARE_LINK_MODE_MASK(advertising); __ETHTOOL_DECLARE_LINK_MODE_MASK(lp_advertising); } link_modes; + u32 lanes; }; /** @@ -265,6 +266,8 @@ struct ethtool_pause_stats { /** * struct ethtool_ops - optional netdev operations + * @cap_link_lanes_supported: indicates if the driver supports lanes + * parameter. * @supported_coalesce_params: supported types of interrupt coalescing. * @get_drvinfo: Report driver/device information. Should only set the * @driver, @version, @fw_version and @bus_info fields. If not @@ -420,6 +423,7 @@ struct ethtool_pause_stats { * of the generic netdev features interface. */ struct ethtool_ops { + u32 cap_link_lanes_supported:1; u32 supported_coalesce_params; void (*get_drvinfo)(struct net_device *, struct ethtool_drvinfo *); int (*get_regs_len)(struct net_device *); diff --git a/include/uapi/linux/ethtool_netlink.h b/include/uapi/linux/ethtool_netlink.h index e2bf36e6964b..a286635ac9b8 100644 --- a/include/uapi/linux/ethtool_netlink.h +++ b/include/uapi/linux/ethtool_netlink.h @@ -227,6 +227,7 @@ enum { ETHTOOL_A_LINKMODES_DUPLEX, /* u8 */ ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG, /* u8 */ ETHTOOL_A_LINKMODES_MASTER_SLAVE_STATE, /* u8 */ + ETHTOOL_A_LINKMODES_LANES, /* u32 */ /* add new constants above here */ __ETHTOOL_A_LINKMODES_CNT, diff --git a/net/ethtool/linkmodes.c b/net/ethtool/linkmodes.c index bb8a3351fb72..db3e31fc6709 100644 --- a/net/ethtool/linkmodes.c +++ b/net/ethtool/linkmodes.c @@ -152,12 +152,47 @@ const struct ethnl_request_ops ethnl_linkmodes_request_ops = { struct link_mode_info { int speed; + u8 lanes; u8 duplex; }; -#define __DEFINE_LINK_MODE_PARAMS(_speed, _type, _duplex) \ - [ETHTOOL_LINK_MODE(_speed, _type, _duplex)] = { \ - .speed = SPEED_ ## _speed, \ +#define __LINK_MODE_LANES_CR 1 +#define __LINK_MODE_LANES_CR2 2 +#define __LINK_MODE_LANES_CR4 4 +#define __LINK_MODE_LANES_CR8 8 +#define __LINK_MODE_LANES_DR 1 +#define __LINK_MODE_LANES_DR2 2 +#define __LINK_MODE_LANES_DR4 4 +#define __LINK_MODE_LANES_DR8 8 +#define __LINK_MODE_LANES_KR 1 +#define __LINK_MODE_LANES_KR2 2 +#define __LINK_MODE_LANES_KR4 4 +#define __LINK_MODE_LANES_KR8 8 +#define __LINK_MODE_LANES_SR 1 +#define __LINK_MODE_LANES_SR2 2 +#define __LINK_MODE_LANES_SR4 4 +#define __LINK_MODE_LANES_SR8 8 +#define __LINK_MODE_LANES_ER 1 +#define __LINK_MODE_LANES_KX 1 +#define __LINK_MODE_LANES_KX4 4 +#define __LINK_MODE_LANES_LR 1 +#define __LINK_MODE_LANES_LR4 4 +#define __LINK_MODE_LANES_LR4_ER4 4 +#define __LINK_MODE_LANES_LR_ER_FR 1 +#define __LINK_MODE_LANES_LR2_ER2_FR2 2 +#define __LINK_MODE_LANES_LR4_ER4_FR4 4 +#define __LINK_MODE_LANES_LR8_ER8_FR8 8 +#define __LINK_MODE_LANES_LRM 1 +#define __LINK_MODE_LANES_MLD2 2 +#define __LINK_MODE_LANES_T 1 +#define __LINK_MODE_LANES_T1 1 +#define __LINK_MODE_LANES_X 1 +#define __LINK_MODE_LANES_FX 1 + +#define __DEFINE_LINK_MODE_PARAMS(_speed, _type, _duplex) \ + [ETHTOOL_LINK_MODE(_speed, _type, _duplex)] = { \ + .speed = SPEED_ ## _speed, \ + .lanes = __LINK_MODE_LANES_ ## _type, \ .duplex = __DUPLEX_ ## _duplex \ } #define __DUPLEX_Half DUPLEX_HALF @@ -165,6 +200,7 @@ struct link_mode_info { #define __DEFINE_SPECIAL_MODE_PARAMS(_mode) \ [ETHTOOL_LINK_MODE_ ## _mode ## _BIT] = { \ .speed = SPEED_UNKNOWN, \ + .lanes = 0, \ .duplex = DUPLEX_UNKNOWN, \ } @@ -274,16 +310,17 @@ const struct nla_policy ethnl_linkmodes_set_policy[] = { [ETHTOOL_A_LINKMODES_SPEED] = { .type = NLA_U32 }, [ETHTOOL_A_LINKMODES_DUPLEX] = { .type = NLA_U8 }, [ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG] = { .type = NLA_U8 }, + [ETHTOOL_A_LINKMODES_LANES] = NLA_POLICY_RANGE(NLA_U32, 1, 8), }; -/* Set advertised link modes to all supported modes matching requested speed - * and duplex values. Called when autonegotiation is on, speed or duplex is - * requested but no link mode change. This is done in userspace with ioctl() - * interface, move it into kernel for netlink. +/* Set advertised link modes to all supported modes matching requested speed, + * lanes and duplex values. Called when autonegotiation is on, speed, lanes or + * duplex is requested but no link mode change. This is done in userspace with + * ioctl() interface, move it into kernel for netlink. * Returns true if advertised modes bitmap was modified. */ static bool ethnl_auto_linkmodes(struct ethtool_link_ksettings *ksettings, - bool req_speed, bool req_duplex) + bool req_speed, bool req_lanes, bool req_duplex) { unsigned long *advertising = ksettings->link_modes.advertising; unsigned long *supported = ksettings->link_modes.supported; @@ -302,6 +339,7 @@ static bool ethnl_auto_linkmodes(struct ethtool_link_ksettings *ksettings, continue; if (test_bit(i, supported) && (!req_speed || info->speed == ksettings->base.speed) && + (!req_lanes || info->lanes == ksettings->lanes) && (!req_duplex || info->duplex == ksettings->base.duplex)) set_bit(i, advertising); else @@ -327,7 +365,7 @@ static bool ethnl_validate_master_slave_cfg(u8 cfg) static int ethnl_check_linkmodes(struct genl_info *info, struct nlattr **tb) { - const struct nlattr *master_slave_cfg; + const struct nlattr *master_slave_cfg, *lanes_cfg; master_slave_cfg = tb[ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG]; if (master_slave_cfg && @@ -337,16 +375,23 @@ static int ethnl_check_linkmodes(struct genl_info *info, struct nlattr **tb) return -EOPNOTSUPP; } + lanes_cfg = tb[ETHTOOL_A_LINKMODES_LANES]; + if (lanes_cfg && !is_power_of_2(nla_get_u32(lanes_cfg))) { + NL_SET_ERR_MSG_ATTR(info->extack, lanes_cfg, + "lanes value is invalid"); + return -EINVAL; + } + return 0; } static int ethnl_update_linkmodes(struct genl_info *info, struct nlattr **tb, struct ethtool_link_ksettings *ksettings, - bool *mod) + bool *mod, const struct net_device *dev) { struct ethtool_link_settings *lsettings = &ksettings->base; - bool req_speed, req_duplex; - const struct nlattr *master_slave_cfg; + bool req_speed, req_lanes, req_duplex; + const struct nlattr *master_slave_cfg, *lanes_cfg; int ret; master_slave_cfg = tb[ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG]; @@ -360,10 +405,30 @@ static int ethnl_update_linkmodes(struct genl_info *info, struct nlattr **tb, *mod = false; req_speed = tb[ETHTOOL_A_LINKMODES_SPEED]; + req_lanes = tb[ETHTOOL_A_LINKMODES_LANES]; req_duplex = tb[ETHTOOL_A_LINKMODES_DUPLEX]; ethnl_update_u8(&lsettings->autoneg, tb[ETHTOOL_A_LINKMODES_AUTONEG], mod); + + lanes_cfg = tb[ETHTOOL_A_LINKMODES_LANES]; + if (lanes_cfg) { + /* If autoneg is off and lanes parameter is not supported by the + * driver, return an error. + */ + if (!lsettings->autoneg && + !dev->ethtool_ops->cap_link_lanes_supported) { + NL_SET_ERR_MSG_ATTR(info->extack, lanes_cfg, + "lanes configuration not supported by device"); + return -EOPNOTSUPP; + } + } else if (!lsettings->autoneg) { + /* If autoneg is off and lanes parameter is not passed from user, + * set the lanes parameter to 0. + */ + ksettings->lanes = 0; + } + ret = ethnl_update_bitset(ksettings->link_modes.advertising, __ETHTOOL_LINK_MODE_MASK_NBITS, tb[ETHTOOL_A_LINKMODES_OURS], link_mode_names, @@ -372,13 +437,14 @@ static int ethnl_update_linkmodes(struct genl_info *info, struct nlattr **tb, return ret; ethnl_update_u32(&lsettings->speed, tb[ETHTOOL_A_LINKMODES_SPEED], mod); + ethnl_update_u32(&ksettings->lanes, lanes_cfg, mod); ethnl_update_u8(&lsettings->duplex, tb[ETHTOOL_A_LINKMODES_DUPLEX], mod); ethnl_update_u8(&lsettings->master_slave_cfg, master_slave_cfg, mod); if (!tb[ETHTOOL_A_LINKMODES_OURS] && lsettings->autoneg && - (req_speed || req_duplex) && - ethnl_auto_linkmodes(ksettings, req_speed, req_duplex)) + (req_speed || req_lanes || req_duplex) && + ethnl_auto_linkmodes(ksettings, req_speed, req_lanes, req_duplex)) *mod = true; return 0; @@ -420,7 +486,7 @@ int ethnl_set_linkmodes(struct sk_buff *skb, struct genl_info *info) goto out_ops; } - ret = ethnl_update_linkmodes(info, tb, &ksettings, &mod); + ret = ethnl_update_linkmodes(info, tb, &ksettings, &mod, dev); if (ret < 0) goto out_ops; diff --git a/net/ethtool/netlink.h b/net/ethtool/netlink.h index d8efec516d86..6eabd58d81bf 100644 --- a/net/ethtool/netlink.h +++ b/net/ethtool/netlink.h @@ -351,7 +351,7 @@ extern const struct nla_policy ethnl_strset_get_policy[ETHTOOL_A_STRSET_COUNTS_O extern const struct nla_policy ethnl_linkinfo_get_policy[ETHTOOL_A_LINKINFO_HEADER + 1]; extern const struct nla_policy ethnl_linkinfo_set_policy[ETHTOOL_A_LINKINFO_TP_MDIX_CTRL + 1]; extern const struct nla_policy ethnl_linkmodes_get_policy[ETHTOOL_A_LINKMODES_HEADER + 1]; -extern const struct nla_policy ethnl_linkmodes_set_policy[ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG + 1]; +extern const struct nla_policy ethnl_linkmodes_set_policy[ETHTOOL_A_LINKMODES_LANES + 1]; extern const struct nla_policy ethnl_linkstate_get_policy[ETHTOOL_A_LINKSTATE_HEADER + 1]; extern const struct nla_policy ethnl_debug_get_policy[ETHTOOL_A_DEBUG_HEADER + 1]; extern const struct nla_policy ethnl_debug_set_policy[ETHTOOL_A_DEBUG_MSGMASK + 1]; -- cgit v1.2.3 From c8907043c6ac9ed58e6c1a76f2824be714b42228 Mon Sep 17 00:00:00 2001 From: Danielle Ratson Date: Tue, 2 Feb 2021 20:06:07 +0200 Subject: ethtool: Get link mode in use instead of speed and duplex parameters Currently, when user space queries the link's parameters, as speed and duplex, each parameter is passed from the driver to ethtool. Instead, get the link mode bit in use, and derive each of the parameters from it in ethtool. Signed-off-by: Danielle Ratson Signed-off-by: Jakub Kicinski --- include/linux/ethtool.h | 1 + net/ethtool/common.c | 147 +++++++++++++++++++++++++++++++++++++++++++++ net/ethtool/common.h | 7 +++ net/ethtool/ioctl.c | 18 +++++- net/ethtool/linkmodes.c | 157 +----------------------------------------------- 5 files changed, 174 insertions(+), 156 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 1ab13c5dfb2f..ec4cd3921c67 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -129,6 +129,7 @@ struct ethtool_link_ksettings { __ETHTOOL_DECLARE_LINK_MODE_MASK(lp_advertising); } link_modes; u32 lanes; + enum ethtool_link_mode_bit_indices link_mode; }; /** diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 181220101a6e..835b9bba3e7e 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -198,6 +198,153 @@ const char link_mode_names[][ETH_GSTRING_LEN] = { }; static_assert(ARRAY_SIZE(link_mode_names) == __ETHTOOL_LINK_MODE_MASK_NBITS); +#define __LINK_MODE_LANES_CR 1 +#define __LINK_MODE_LANES_CR2 2 +#define __LINK_MODE_LANES_CR4 4 +#define __LINK_MODE_LANES_CR8 8 +#define __LINK_MODE_LANES_DR 1 +#define __LINK_MODE_LANES_DR2 2 +#define __LINK_MODE_LANES_DR4 4 +#define __LINK_MODE_LANES_DR8 8 +#define __LINK_MODE_LANES_KR 1 +#define __LINK_MODE_LANES_KR2 2 +#define __LINK_MODE_LANES_KR4 4 +#define __LINK_MODE_LANES_KR8 8 +#define __LINK_MODE_LANES_SR 1 +#define __LINK_MODE_LANES_SR2 2 +#define __LINK_MODE_LANES_SR4 4 +#define __LINK_MODE_LANES_SR8 8 +#define __LINK_MODE_LANES_ER 1 +#define __LINK_MODE_LANES_KX 1 +#define __LINK_MODE_LANES_KX4 4 +#define __LINK_MODE_LANES_LR 1 +#define __LINK_MODE_LANES_LR4 4 +#define __LINK_MODE_LANES_LR4_ER4 4 +#define __LINK_MODE_LANES_LR_ER_FR 1 +#define __LINK_MODE_LANES_LR2_ER2_FR2 2 +#define __LINK_MODE_LANES_LR4_ER4_FR4 4 +#define __LINK_MODE_LANES_LR8_ER8_FR8 8 +#define __LINK_MODE_LANES_LRM 1 +#define __LINK_MODE_LANES_MLD2 2 +#define __LINK_MODE_LANES_T 1 +#define __LINK_MODE_LANES_T1 1 +#define __LINK_MODE_LANES_X 1 +#define __LINK_MODE_LANES_FX 1 + +#define __DEFINE_LINK_MODE_PARAMS(_speed, _type, _duplex) \ + [ETHTOOL_LINK_MODE(_speed, _type, _duplex)] = { \ + .speed = SPEED_ ## _speed, \ + .lanes = __LINK_MODE_LANES_ ## _type, \ + .duplex = __DUPLEX_ ## _duplex \ + } +#define __DUPLEX_Half DUPLEX_HALF +#define __DUPLEX_Full DUPLEX_FULL +#define __DEFINE_SPECIAL_MODE_PARAMS(_mode) \ + [ETHTOOL_LINK_MODE_ ## _mode ## _BIT] = { \ + .speed = SPEED_UNKNOWN, \ + .lanes = 0, \ + .duplex = DUPLEX_UNKNOWN, \ + } + +const struct link_mode_info link_mode_params[] = { + __DEFINE_LINK_MODE_PARAMS(10, T, Half), + __DEFINE_LINK_MODE_PARAMS(10, T, Full), + __DEFINE_LINK_MODE_PARAMS(100, T, Half), + __DEFINE_LINK_MODE_PARAMS(100, T, Full), + __DEFINE_LINK_MODE_PARAMS(1000, T, Half), + __DEFINE_LINK_MODE_PARAMS(1000, T, Full), + __DEFINE_SPECIAL_MODE_PARAMS(Autoneg), + __DEFINE_SPECIAL_MODE_PARAMS(TP), + __DEFINE_SPECIAL_MODE_PARAMS(AUI), + __DEFINE_SPECIAL_MODE_PARAMS(MII), + __DEFINE_SPECIAL_MODE_PARAMS(FIBRE), + __DEFINE_SPECIAL_MODE_PARAMS(BNC), + __DEFINE_LINK_MODE_PARAMS(10000, T, Full), + __DEFINE_SPECIAL_MODE_PARAMS(Pause), + __DEFINE_SPECIAL_MODE_PARAMS(Asym_Pause), + __DEFINE_LINK_MODE_PARAMS(2500, X, Full), + __DEFINE_SPECIAL_MODE_PARAMS(Backplane), + __DEFINE_LINK_MODE_PARAMS(1000, KX, Full), + __DEFINE_LINK_MODE_PARAMS(10000, KX4, Full), + __DEFINE_LINK_MODE_PARAMS(10000, KR, Full), + [ETHTOOL_LINK_MODE_10000baseR_FEC_BIT] = { + .speed = SPEED_10000, + .duplex = DUPLEX_FULL, + }, + __DEFINE_LINK_MODE_PARAMS(20000, MLD2, Full), + __DEFINE_LINK_MODE_PARAMS(20000, KR2, Full), + __DEFINE_LINK_MODE_PARAMS(40000, KR4, Full), + __DEFINE_LINK_MODE_PARAMS(40000, CR4, Full), + __DEFINE_LINK_MODE_PARAMS(40000, SR4, Full), + __DEFINE_LINK_MODE_PARAMS(40000, LR4, Full), + __DEFINE_LINK_MODE_PARAMS(56000, KR4, Full), + __DEFINE_LINK_MODE_PARAMS(56000, CR4, Full), + __DEFINE_LINK_MODE_PARAMS(56000, SR4, Full), + __DEFINE_LINK_MODE_PARAMS(56000, LR4, Full), + __DEFINE_LINK_MODE_PARAMS(25000, CR, Full), + __DEFINE_LINK_MODE_PARAMS(25000, KR, Full), + __DEFINE_LINK_MODE_PARAMS(25000, SR, Full), + __DEFINE_LINK_MODE_PARAMS(50000, CR2, Full), + __DEFINE_LINK_MODE_PARAMS(50000, KR2, Full), + __DEFINE_LINK_MODE_PARAMS(100000, KR4, Full), + __DEFINE_LINK_MODE_PARAMS(100000, SR4, Full), + __DEFINE_LINK_MODE_PARAMS(100000, CR4, Full), + __DEFINE_LINK_MODE_PARAMS(100000, LR4_ER4, Full), + __DEFINE_LINK_MODE_PARAMS(50000, SR2, Full), + __DEFINE_LINK_MODE_PARAMS(1000, X, Full), + __DEFINE_LINK_MODE_PARAMS(10000, CR, Full), + __DEFINE_LINK_MODE_PARAMS(10000, SR, Full), + __DEFINE_LINK_MODE_PARAMS(10000, LR, Full), + __DEFINE_LINK_MODE_PARAMS(10000, LRM, Full), + __DEFINE_LINK_MODE_PARAMS(10000, ER, Full), + __DEFINE_LINK_MODE_PARAMS(2500, T, Full), + __DEFINE_LINK_MODE_PARAMS(5000, T, Full), + __DEFINE_SPECIAL_MODE_PARAMS(FEC_NONE), + __DEFINE_SPECIAL_MODE_PARAMS(FEC_RS), + __DEFINE_SPECIAL_MODE_PARAMS(FEC_BASER), + __DEFINE_LINK_MODE_PARAMS(50000, KR, Full), + __DEFINE_LINK_MODE_PARAMS(50000, SR, Full), + __DEFINE_LINK_MODE_PARAMS(50000, CR, Full), + __DEFINE_LINK_MODE_PARAMS(50000, LR_ER_FR, Full), + __DEFINE_LINK_MODE_PARAMS(50000, DR, Full), + __DEFINE_LINK_MODE_PARAMS(100000, KR2, Full), + __DEFINE_LINK_MODE_PARAMS(100000, SR2, Full), + __DEFINE_LINK_MODE_PARAMS(100000, CR2, Full), + __DEFINE_LINK_MODE_PARAMS(100000, LR2_ER2_FR2, Full), + __DEFINE_LINK_MODE_PARAMS(100000, DR2, Full), + __DEFINE_LINK_MODE_PARAMS(200000, KR4, Full), + __DEFINE_LINK_MODE_PARAMS(200000, SR4, Full), + __DEFINE_LINK_MODE_PARAMS(200000, LR4_ER4_FR4, Full), + __DEFINE_LINK_MODE_PARAMS(200000, DR4, Full), + __DEFINE_LINK_MODE_PARAMS(200000, CR4, Full), + __DEFINE_LINK_MODE_PARAMS(100, T1, Full), + __DEFINE_LINK_MODE_PARAMS(1000, T1, Full), + __DEFINE_LINK_MODE_PARAMS(400000, KR8, Full), + __DEFINE_LINK_MODE_PARAMS(400000, SR8, Full), + __DEFINE_LINK_MODE_PARAMS(400000, LR8_ER8_FR8, Full), + __DEFINE_LINK_MODE_PARAMS(400000, DR8, Full), + __DEFINE_LINK_MODE_PARAMS(400000, CR8, Full), + __DEFINE_SPECIAL_MODE_PARAMS(FEC_LLRS), + __DEFINE_LINK_MODE_PARAMS(100000, KR, Full), + __DEFINE_LINK_MODE_PARAMS(100000, SR, Full), + __DEFINE_LINK_MODE_PARAMS(100000, LR_ER_FR, Full), + __DEFINE_LINK_MODE_PARAMS(100000, DR, Full), + __DEFINE_LINK_MODE_PARAMS(100000, CR, Full), + __DEFINE_LINK_MODE_PARAMS(200000, KR2, Full), + __DEFINE_LINK_MODE_PARAMS(200000, SR2, Full), + __DEFINE_LINK_MODE_PARAMS(200000, LR2_ER2_FR2, Full), + __DEFINE_LINK_MODE_PARAMS(200000, DR2, Full), + __DEFINE_LINK_MODE_PARAMS(200000, CR2, Full), + __DEFINE_LINK_MODE_PARAMS(400000, KR4, Full), + __DEFINE_LINK_MODE_PARAMS(400000, SR4, Full), + __DEFINE_LINK_MODE_PARAMS(400000, LR4_ER4_FR4, Full), + __DEFINE_LINK_MODE_PARAMS(400000, DR4, Full), + __DEFINE_LINK_MODE_PARAMS(400000, CR4, Full), + __DEFINE_LINK_MODE_PARAMS(100, FX, Half), + __DEFINE_LINK_MODE_PARAMS(100, FX, Full), +}; +static_assert(ARRAY_SIZE(link_mode_params) == __ETHTOOL_LINK_MODE_MASK_NBITS); + const char netif_msg_class_names[][ETH_GSTRING_LEN] = { [NETIF_MSG_DRV_BIT] = "drv", [NETIF_MSG_PROBE_BIT] = "probe", diff --git a/net/ethtool/common.h b/net/ethtool/common.h index 3d9251c95a8b..a9d071248698 100644 --- a/net/ethtool/common.h +++ b/net/ethtool/common.h @@ -14,6 +14,12 @@ #define __SOF_TIMESTAMPING_CNT (const_ilog2(SOF_TIMESTAMPING_LAST) + 1) +struct link_mode_info { + int speed; + u8 lanes; + u8 duplex; +}; + extern const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN]; extern const char @@ -23,6 +29,7 @@ tunable_strings[__ETHTOOL_TUNABLE_COUNT][ETH_GSTRING_LEN]; extern const char phy_tunable_strings[__ETHTOOL_PHY_TUNABLE_COUNT][ETH_GSTRING_LEN]; extern const char link_mode_names[][ETH_GSTRING_LEN]; +extern const struct link_mode_info link_mode_params[]; extern const char netif_msg_class_names[][ETH_GSTRING_LEN]; extern const char wol_mode_names[][ETH_GSTRING_LEN]; extern const char sof_timestamping_names[][ETH_GSTRING_LEN]; diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 771688e1b0da..24783b71c584 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -426,13 +426,29 @@ struct ethtool_link_usettings { int __ethtool_get_link_ksettings(struct net_device *dev, struct ethtool_link_ksettings *link_ksettings) { + const struct link_mode_info *link_info; + int err; + ASSERT_RTNL(); if (!dev->ethtool_ops->get_link_ksettings) return -EOPNOTSUPP; memset(link_ksettings, 0, sizeof(*link_ksettings)); - return dev->ethtool_ops->get_link_ksettings(dev, link_ksettings); + + link_ksettings->link_mode = -1; + err = dev->ethtool_ops->get_link_ksettings(dev, link_ksettings); + if (err) + return err; + + if (link_ksettings->link_mode != -1) { + link_info = &link_mode_params[link_ksettings->link_mode]; + link_ksettings->base.speed = link_info->speed; + link_ksettings->lanes = link_info->lanes; + link_ksettings->base.duplex = link_info->duplex; + } + + return 0; } EXPORT_SYMBOL(__ethtool_get_link_ksettings); diff --git a/net/ethtool/linkmodes.c b/net/ethtool/linkmodes.c index db3e31fc6709..fc986d035b01 100644 --- a/net/ethtool/linkmodes.c +++ b/net/ethtool/linkmodes.c @@ -4,6 +4,8 @@ #include "common.h" #include "bitset.h" +/* LINKMODES_GET */ + struct linkmodes_req_info { struct ethnl_req_info base; }; @@ -150,158 +152,6 @@ const struct ethnl_request_ops ethnl_linkmodes_request_ops = { /* LINKMODES_SET */ -struct link_mode_info { - int speed; - u8 lanes; - u8 duplex; -}; - -#define __LINK_MODE_LANES_CR 1 -#define __LINK_MODE_LANES_CR2 2 -#define __LINK_MODE_LANES_CR4 4 -#define __LINK_MODE_LANES_CR8 8 -#define __LINK_MODE_LANES_DR 1 -#define __LINK_MODE_LANES_DR2 2 -#define __LINK_MODE_LANES_DR4 4 -#define __LINK_MODE_LANES_DR8 8 -#define __LINK_MODE_LANES_KR 1 -#define __LINK_MODE_LANES_KR2 2 -#define __LINK_MODE_LANES_KR4 4 -#define __LINK_MODE_LANES_KR8 8 -#define __LINK_MODE_LANES_SR 1 -#define __LINK_MODE_LANES_SR2 2 -#define __LINK_MODE_LANES_SR4 4 -#define __LINK_MODE_LANES_SR8 8 -#define __LINK_MODE_LANES_ER 1 -#define __LINK_MODE_LANES_KX 1 -#define __LINK_MODE_LANES_KX4 4 -#define __LINK_MODE_LANES_LR 1 -#define __LINK_MODE_LANES_LR4 4 -#define __LINK_MODE_LANES_LR4_ER4 4 -#define __LINK_MODE_LANES_LR_ER_FR 1 -#define __LINK_MODE_LANES_LR2_ER2_FR2 2 -#define __LINK_MODE_LANES_LR4_ER4_FR4 4 -#define __LINK_MODE_LANES_LR8_ER8_FR8 8 -#define __LINK_MODE_LANES_LRM 1 -#define __LINK_MODE_LANES_MLD2 2 -#define __LINK_MODE_LANES_T 1 -#define __LINK_MODE_LANES_T1 1 -#define __LINK_MODE_LANES_X 1 -#define __LINK_MODE_LANES_FX 1 - -#define __DEFINE_LINK_MODE_PARAMS(_speed, _type, _duplex) \ - [ETHTOOL_LINK_MODE(_speed, _type, _duplex)] = { \ - .speed = SPEED_ ## _speed, \ - .lanes = __LINK_MODE_LANES_ ## _type, \ - .duplex = __DUPLEX_ ## _duplex \ - } -#define __DUPLEX_Half DUPLEX_HALF -#define __DUPLEX_Full DUPLEX_FULL -#define __DEFINE_SPECIAL_MODE_PARAMS(_mode) \ - [ETHTOOL_LINK_MODE_ ## _mode ## _BIT] = { \ - .speed = SPEED_UNKNOWN, \ - .lanes = 0, \ - .duplex = DUPLEX_UNKNOWN, \ - } - -static const struct link_mode_info link_mode_params[] = { - __DEFINE_LINK_MODE_PARAMS(10, T, Half), - __DEFINE_LINK_MODE_PARAMS(10, T, Full), - __DEFINE_LINK_MODE_PARAMS(100, T, Half), - __DEFINE_LINK_MODE_PARAMS(100, T, Full), - __DEFINE_LINK_MODE_PARAMS(1000, T, Half), - __DEFINE_LINK_MODE_PARAMS(1000, T, Full), - __DEFINE_SPECIAL_MODE_PARAMS(Autoneg), - __DEFINE_SPECIAL_MODE_PARAMS(TP), - __DEFINE_SPECIAL_MODE_PARAMS(AUI), - __DEFINE_SPECIAL_MODE_PARAMS(MII), - __DEFINE_SPECIAL_MODE_PARAMS(FIBRE), - __DEFINE_SPECIAL_MODE_PARAMS(BNC), - __DEFINE_LINK_MODE_PARAMS(10000, T, Full), - __DEFINE_SPECIAL_MODE_PARAMS(Pause), - __DEFINE_SPECIAL_MODE_PARAMS(Asym_Pause), - __DEFINE_LINK_MODE_PARAMS(2500, X, Full), - __DEFINE_SPECIAL_MODE_PARAMS(Backplane), - __DEFINE_LINK_MODE_PARAMS(1000, KX, Full), - __DEFINE_LINK_MODE_PARAMS(10000, KX4, Full), - __DEFINE_LINK_MODE_PARAMS(10000, KR, Full), - [ETHTOOL_LINK_MODE_10000baseR_FEC_BIT] = { - .speed = SPEED_10000, - .duplex = DUPLEX_FULL, - }, - __DEFINE_LINK_MODE_PARAMS(20000, MLD2, Full), - __DEFINE_LINK_MODE_PARAMS(20000, KR2, Full), - __DEFINE_LINK_MODE_PARAMS(40000, KR4, Full), - __DEFINE_LINK_MODE_PARAMS(40000, CR4, Full), - __DEFINE_LINK_MODE_PARAMS(40000, SR4, Full), - __DEFINE_LINK_MODE_PARAMS(40000, LR4, Full), - __DEFINE_LINK_MODE_PARAMS(56000, KR4, Full), - __DEFINE_LINK_MODE_PARAMS(56000, CR4, Full), - __DEFINE_LINK_MODE_PARAMS(56000, SR4, Full), - __DEFINE_LINK_MODE_PARAMS(56000, LR4, Full), - __DEFINE_LINK_MODE_PARAMS(25000, CR, Full), - __DEFINE_LINK_MODE_PARAMS(25000, KR, Full), - __DEFINE_LINK_MODE_PARAMS(25000, SR, Full), - __DEFINE_LINK_MODE_PARAMS(50000, CR2, Full), - __DEFINE_LINK_MODE_PARAMS(50000, KR2, Full), - __DEFINE_LINK_MODE_PARAMS(100000, KR4, Full), - __DEFINE_LINK_MODE_PARAMS(100000, SR4, Full), - __DEFINE_LINK_MODE_PARAMS(100000, CR4, Full), - __DEFINE_LINK_MODE_PARAMS(100000, LR4_ER4, Full), - __DEFINE_LINK_MODE_PARAMS(50000, SR2, Full), - __DEFINE_LINK_MODE_PARAMS(1000, X, Full), - __DEFINE_LINK_MODE_PARAMS(10000, CR, Full), - __DEFINE_LINK_MODE_PARAMS(10000, SR, Full), - __DEFINE_LINK_MODE_PARAMS(10000, LR, Full), - __DEFINE_LINK_MODE_PARAMS(10000, LRM, Full), - __DEFINE_LINK_MODE_PARAMS(10000, ER, Full), - __DEFINE_LINK_MODE_PARAMS(2500, T, Full), - __DEFINE_LINK_MODE_PARAMS(5000, T, Full), - __DEFINE_SPECIAL_MODE_PARAMS(FEC_NONE), - __DEFINE_SPECIAL_MODE_PARAMS(FEC_RS), - __DEFINE_SPECIAL_MODE_PARAMS(FEC_BASER), - __DEFINE_LINK_MODE_PARAMS(50000, KR, Full), - __DEFINE_LINK_MODE_PARAMS(50000, SR, Full), - __DEFINE_LINK_MODE_PARAMS(50000, CR, Full), - __DEFINE_LINK_MODE_PARAMS(50000, LR_ER_FR, Full), - __DEFINE_LINK_MODE_PARAMS(50000, DR, Full), - __DEFINE_LINK_MODE_PARAMS(100000, KR2, Full), - __DEFINE_LINK_MODE_PARAMS(100000, SR2, Full), - __DEFINE_LINK_MODE_PARAMS(100000, CR2, Full), - __DEFINE_LINK_MODE_PARAMS(100000, LR2_ER2_FR2, Full), - __DEFINE_LINK_MODE_PARAMS(100000, DR2, Full), - __DEFINE_LINK_MODE_PARAMS(200000, KR4, Full), - __DEFINE_LINK_MODE_PARAMS(200000, SR4, Full), - __DEFINE_LINK_MODE_PARAMS(200000, LR4_ER4_FR4, Full), - __DEFINE_LINK_MODE_PARAMS(200000, DR4, Full), - __DEFINE_LINK_MODE_PARAMS(200000, CR4, Full), - __DEFINE_LINK_MODE_PARAMS(100, T1, Full), - __DEFINE_LINK_MODE_PARAMS(1000, T1, Full), - __DEFINE_LINK_MODE_PARAMS(400000, KR8, Full), - __DEFINE_LINK_MODE_PARAMS(400000, SR8, Full), - __DEFINE_LINK_MODE_PARAMS(400000, LR8_ER8_FR8, Full), - __DEFINE_LINK_MODE_PARAMS(400000, DR8, Full), - __DEFINE_LINK_MODE_PARAMS(400000, CR8, Full), - __DEFINE_SPECIAL_MODE_PARAMS(FEC_LLRS), - __DEFINE_LINK_MODE_PARAMS(100000, KR, Full), - __DEFINE_LINK_MODE_PARAMS(100000, SR, Full), - __DEFINE_LINK_MODE_PARAMS(100000, LR_ER_FR, Full), - __DEFINE_LINK_MODE_PARAMS(100000, DR, Full), - __DEFINE_LINK_MODE_PARAMS(100000, CR, Full), - __DEFINE_LINK_MODE_PARAMS(200000, KR2, Full), - __DEFINE_LINK_MODE_PARAMS(200000, SR2, Full), - __DEFINE_LINK_MODE_PARAMS(200000, LR2_ER2_FR2, Full), - __DEFINE_LINK_MODE_PARAMS(200000, DR2, Full), - __DEFINE_LINK_MODE_PARAMS(200000, CR2, Full), - __DEFINE_LINK_MODE_PARAMS(400000, KR4, Full), - __DEFINE_LINK_MODE_PARAMS(400000, SR4, Full), - __DEFINE_LINK_MODE_PARAMS(400000, LR4_ER4_FR4, Full), - __DEFINE_LINK_MODE_PARAMS(400000, DR4, Full), - __DEFINE_LINK_MODE_PARAMS(400000, CR4, Full), - __DEFINE_LINK_MODE_PARAMS(100, FX, Half), - __DEFINE_LINK_MODE_PARAMS(100, FX, Full), -}; - const struct nla_policy ethnl_linkmodes_set_policy[] = { [ETHTOOL_A_LINKMODES_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), @@ -327,9 +177,6 @@ static bool ethnl_auto_linkmodes(struct ethtool_link_ksettings *ksettings, DECLARE_BITMAP(old_adv, __ETHTOOL_LINK_MODE_MASK_NBITS); unsigned int i; - BUILD_BUG_ON(ARRAY_SIZE(link_mode_params) != - __ETHTOOL_LINK_MODE_MASK_NBITS); - bitmap_copy(old_adv, advertising, __ETHTOOL_LINK_MODE_MASK_NBITS); for (i = 0; i < __ETHTOOL_LINK_MODE_MASK_NBITS; i++) { -- cgit v1.2.3 From 2a1673f0f1de78b146bfdbe8c8a773c4a0499790 Mon Sep 17 00:00:00 2001 From: Kyle Tso Date: Thu, 4 Feb 2021 08:50:36 +0800 Subject: usb: pd: Reland VDO definitions of PD2.0 Reland VDO definitions of PD Revision 2.0 as they are still used in PD2.0 products. Fixes: 0e1d6f55a12e ("usb: pd: Update VDO definitions") Signed-off-by: Kyle Tso Link: https://lore.kernel.org/r/20210204005036.1555294-1-kyletso@google.com Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/pd_vdo.h | 69 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/usb/pd_vdo.h b/include/linux/usb/pd_vdo.h index e9b6822c54c2..5de7f550f93e 100644 --- a/include/linux/usb/pd_vdo.h +++ b/include/linux/usb/pd_vdo.h @@ -110,6 +110,10 @@ * <20:16> :: Reserved, Shall be set to zero * <15:0> :: USB-IF assigned VID for this cable vendor */ + +/* PD Rev2.0 definition */ +#define IDH_PTYPE_UNDEF 0 + /* SOP Product Type (UFP) */ #define IDH_PTYPE_NOT_UFP 0 #define IDH_PTYPE_HUB 1 @@ -248,7 +252,25 @@ | ((pnum) & 0x1f)) /* - * Passive Cable VDO + * Cable VDO (for both Passive and Active Cable VDO in PD Rev2.0) + * --------- + * <31:28> :: Cable HW version + * <27:24> :: Cable FW version + * <23:20> :: Reserved, Shall be set to zero + * <19:18> :: type-C to Type-A/B/C/Captive (00b == A, 01 == B, 10 == C, 11 == Captive) + * <17> :: Reserved, Shall be set to zero + * <16:13> :: cable latency (0001 == <10ns(~1m length)) + * <12:11> :: cable termination type (11b == both ends active VCONN req) + * <10> :: SSTX1 Directionality support (0b == fixed, 1b == cfgable) + * <9> :: SSTX2 Directionality support + * <8> :: SSRX1 Directionality support + * <7> :: SSRX2 Directionality support + * <6:5> :: Vbus current handling capability (01b == 3A, 10b == 5A) + * <4> :: Vbus through cable (0b == no, 1b == yes) + * <3> :: SOP" controller present? (0b == no, 1b == yes) + * <2:0> :: USB SS Signaling support + * + * Passive Cable VDO (PD Rev3.0+) * --------- * <31:28> :: Cable HW version * <27:24> :: Cable FW version @@ -264,7 +286,7 @@ * <4:3> :: Reserved, Shall be set to zero * <2:0> :: USB highest speed * - * Active Cable VDO 1 + * Active Cable VDO 1 (PD Rev3.0+) * --------- * <31:28> :: Cable HW version * <27:24> :: Cable FW version @@ -286,7 +308,9 @@ #define CABLE_VDO_VER1_0 0 #define CABLE_VDO_VER1_3 3 -/* Connector Type */ +/* Connector Type (_ATYPE and _BTYPE are for PD Rev2.0 only) */ +#define CABLE_ATYPE 0 +#define CABLE_BTYPE 1 #define CABLE_CTYPE 2 #define CABLE_CAPTIVE 3 @@ -323,12 +347,22 @@ #define CABLE_CURR_3A 1 #define CABLE_CURR_5A 2 +/* USB SuperSpeed Signaling Support (PD Rev2.0) */ +#define CABLE_USBSS_U2_ONLY 0 +#define CABLE_USBSS_U31_GEN1 1 +#define CABLE_USBSS_U31_GEN2 2 + /* USB Highest Speed */ #define CABLE_USB2_ONLY 0 #define CABLE_USB32_GEN1 1 #define CABLE_USB32_4_GEN2 2 #define CABLE_USB4_GEN3 3 +#define VDO_CABLE(hw, fw, cbl, lat, term, tx1d, tx2d, rx1d, rx2d, cur, vps, sopp, usbss) \ + (((hw) & 0x7) << 28 | ((fw) & 0x7) << 24 | ((cbl) & 0x3) << 18 \ + | ((lat) & 0x7) << 13 | ((term) & 0x3) << 11 | (tx1d) << 10 \ + | (tx2d) << 9 | (rx1d) << 8 | (rx2d) << 7 | ((cur) & 0x3) << 5 \ + | (vps) << 4 | (sopp) << 3 | ((usbss) & 0x7)) #define VDO_PCABLE(hw, fw, ver, conn, lat, term, vbm, cur, spd) \ (((hw) & 0xf) << 28 | ((fw) & 0xf) << 24 | ((ver) & 0x7) << 21 \ | ((conn) & 0x3) << 18 | ((lat) & 0xf) << 13 | ((term) & 0x3) << 11 \ @@ -395,6 +429,35 @@ | ((hops) & 0x3) << 6 | (u2) << 5 | (u32) << 4 | (lane) << 3 \ | (iso) << 2 | (gen)) +/* + * AMA VDO (PD Rev2.0) + * --------- + * <31:28> :: Cable HW version + * <27:24> :: Cable FW version + * <23:12> :: Reserved, Shall be set to zero + * <11> :: SSTX1 Directionality support (0b == fixed, 1b == cfgable) + * <10> :: SSTX2 Directionality support + * <9> :: SSRX1 Directionality support + * <8> :: SSRX2 Directionality support + * <7:5> :: Vconn power + * <4> :: Vconn power required + * <3> :: Vbus power required + * <2:0> :: USB SS Signaling support + */ +#define VDO_AMA(hw, fw, tx1d, tx2d, rx1d, rx2d, vcpwr, vcr, vbr, usbss) \ + (((hw) & 0x7) << 28 | ((fw) & 0x7) << 24 \ + | (tx1d) << 11 | (tx2d) << 10 | (rx1d) << 9 | (rx2d) << 8 \ + | ((vcpwr) & 0x7) << 5 | (vcr) << 4 | (vbr) << 3 \ + | ((usbss) & 0x7)) + +#define PD_VDO_AMA_VCONN_REQ(vdo) (((vdo) >> 4) & 1) +#define PD_VDO_AMA_VBUS_REQ(vdo) (((vdo) >> 3) & 1) + +#define AMA_USBSS_U2_ONLY 0 +#define AMA_USBSS_U31_GEN1 1 +#define AMA_USBSS_U31_GEN2 2 +#define AMA_USBSS_BBONLY 3 + /* * VPD VDO * --------- -- cgit v1.2.3 From 3cd542e6e6afb6fa6c34d4094d498f42e22110f5 Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Thu, 3 Sep 2020 13:13:21 +0300 Subject: thunderbolt: Add support for PCIe tunneling disabled (SL5) Recent Intel Thunderbolt firmware connection manager has support for another security level, SL5, that disables PCIe tunneling. This option can be turned on from the BIOS. When this is set the driver exposes a new security level "nopcie" to the userspace and hides the authorized attribute under connected devices. While there we also hide it when "dponly" security level is enabled since it is not really usable in that case anyway. Signed-off-by: Mika Westerberg Acked-by: Yehezkel Bernat --- Documentation/ABI/testing/sysfs-bus-thunderbolt | 2 ++ Documentation/admin-guide/thunderbolt.rst | 7 +++++++ drivers/thunderbolt/domain.c | 12 +++++++++++- drivers/thunderbolt/switch.c | 6 +++++- include/linux/thunderbolt.h | 3 +++ 5 files changed, 28 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-bus-thunderbolt b/Documentation/ABI/testing/sysfs-bus-thunderbolt index 581dea95245b..d7f09d011b6d 100644 --- a/Documentation/ABI/testing/sysfs-bus-thunderbolt +++ b/Documentation/ABI/testing/sysfs-bus-thunderbolt @@ -85,6 +85,8 @@ Description: This attribute holds current Thunderbolt security level usbonly Automatically tunnel USB controller of the connected Thunderbolt dock (and Display Port). All PCIe links downstream of the dock are removed. + nopcie USB4 system where PCIe tunneling is disabled from + the BIOS. ======= ================================================== What: /sys/bus/thunderbolt/devices/.../authorized diff --git a/Documentation/admin-guide/thunderbolt.rst b/Documentation/admin-guide/thunderbolt.rst index 0d4348445f91..f18e881373c4 100644 --- a/Documentation/admin-guide/thunderbolt.rst +++ b/Documentation/admin-guide/thunderbolt.rst @@ -47,6 +47,9 @@ be DMA masters and thus read contents of the host memory without CPU and OS knowing about it. There are ways to prevent this by setting up an IOMMU but it is not always available for various reasons. +Some USB4 systems have a BIOS setting to disable PCIe tunneling. This is +treated as another security level (nopcie). + The security levels are as follows: none @@ -77,6 +80,10 @@ The security levels are as follows: Display Port in a dock. All PCIe links downstream of the dock are removed. + nopcie + PCIe tunneling is disabled/forbidden from the BIOS. Available in some + USB4 systems. + The current security level can be read from ``/sys/bus/thunderbolt/devices/domainX/security`` where ``domainX`` is the Thunderbolt domain the host controller manages. There is typically diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c index 9ba2181464cc..a1c79c9c4f66 100644 --- a/drivers/thunderbolt/domain.c +++ b/drivers/thunderbolt/domain.c @@ -118,6 +118,7 @@ static const char * const tb_security_names[] = { [TB_SECURITY_SECURE] = "secure", [TB_SECURITY_DPONLY] = "dponly", [TB_SECURITY_USBONLY] = "usbonly", + [TB_SECURITY_NOPCIE] = "nopcie", }; static ssize_t boot_acl_show(struct device *dev, struct device_attribute *attr, @@ -243,8 +244,14 @@ static ssize_t deauthorization_show(struct device *dev, char *buf) { const struct tb *tb = container_of(dev, struct tb, dev); + bool deauthorization = false; - return sprintf(buf, "%d\n", !!tb->cm_ops->disapprove_switch); + /* Only meaningful if authorization is supported */ + if (tb->security_level == TB_SECURITY_USER || + tb->security_level == TB_SECURITY_SECURE) + deauthorization = !!tb->cm_ops->disapprove_switch; + + return sprintf(buf, "%d\n", deauthorization); } static DEVICE_ATTR_RO(deauthorization); @@ -452,6 +459,9 @@ int tb_domain_add(struct tb *tb) goto err_ctl_stop; } + tb_dbg(tb, "security level set to %s\n", + tb_security_names[tb->security_level]); + ret = device_add(&tb->dev); if (ret) goto err_ctl_stop; diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index 5377d0a3390f..b63fecca6c2a 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -1774,7 +1774,11 @@ static umode_t switch_attr_is_visible(struct kobject *kobj, struct device *dev = kobj_to_dev(kobj); struct tb_switch *sw = tb_to_switch(dev); - if (attr == &dev_attr_device.attr) { + if (attr == &dev_attr_authorized.attr) { + if (sw->tb->security_level == TB_SECURITY_NOPCIE || + sw->tb->security_level == TB_SECURITY_DPONLY) + return 0; + } else if (attr == &dev_attr_device.attr) { if (!sw->device) return 0; } else if (attr == &dev_attr_device_name.attr) { diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h index 034dccf93955..659a0a810fa1 100644 --- a/include/linux/thunderbolt.h +++ b/include/linux/thunderbolt.h @@ -45,6 +45,8 @@ enum tb_cfg_pkg_type { * @TB_SECURITY_USBONLY: Only tunnel USB controller of the connected * Thunderbolt dock (and Display Port). All PCIe * links downstream of the dock are removed. + * @TB_SECURITY_NOPCIE: For USB4 systems this level is used when the + * PCIe tunneling is disabled from the BIOS. */ enum tb_security_level { TB_SECURITY_NONE, @@ -52,6 +54,7 @@ enum tb_security_level { TB_SECURITY_SECURE, TB_SECURITY_DPONLY, TB_SECURITY_USBONLY, + TB_SECURITY_NOPCIE, }; /** -- cgit v1.2.3 From 5a6a2c0f0f43676df27632d657a3f18b151a7ef8 Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Tue, 18 Feb 2020 16:02:45 +0200 Subject: ACPI: Add support for native USB4 control _OSC ACPI 6.4 introduced a new _OSC capability that is used negotiate native connection manager support. Connection manager is the entity that is responsible for tunneling over the USB4 fabric. If the platform rejects the native access then firmware based connection manager is used. The new _OSC also includes a set of bits that can be used to disable certain tunnel types such as PCIe for security reasons for instance. This implements the new USB4 _OSC so that we try to negotiate native USB4 support if the Thunderbolt/USB4 (CONFIG_USB4) driver is enabled. Drivers can determine what was negotiated by checking two new variables exposed in this patch. Signed-off-by: Mika Westerberg Reviewed-by: Rafael J. Wysocki --- drivers/acpi/bus.c | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/acpi.h | 10 +++++++ 2 files changed, 86 insertions(+) (limited to 'include/linux') diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index a52cb28c40d8..9c3fe08e8f18 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -281,6 +281,12 @@ bool osc_sb_apei_support_acked; bool osc_pc_lpi_support_confirmed; EXPORT_SYMBOL_GPL(osc_pc_lpi_support_confirmed); +/* + * ACPI 6.4 Operating System Capabilities for USB. + */ +bool osc_sb_native_usb4_support_confirmed; +EXPORT_SYMBOL_GPL(osc_sb_native_usb4_support_confirmed); + static u8 sb_uuid_str[] = "0811B06E-4A27-44F9-8D60-3CBBC22E7B48"; static void acpi_bus_osc_negotiate_platform_control(void) { @@ -317,6 +323,9 @@ static void acpi_bus_osc_negotiate_platform_control(void) if (IS_ENABLED(CONFIG_SCHED_MC_PRIO)) capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_CPC_DIVERSE_HIGH_SUPPORT; + if (IS_ENABLED(CONFIG_USB4)) + capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_NATIVE_USB4_SUPPORT; + if (!ghes_disable) capbuf[OSC_SUPPORT_DWORD] |= OSC_SB_APEI_SUPPORT; if (ACPI_FAILURE(acpi_get_handle(NULL, "\\_SB", &handle))) @@ -348,8 +357,74 @@ static void acpi_bus_osc_negotiate_platform_control(void) capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_APEI_SUPPORT; osc_pc_lpi_support_confirmed = capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_PCLPI_SUPPORT; + osc_sb_native_usb4_support_confirmed = + capbuf_ret[OSC_SUPPORT_DWORD] & OSC_SB_NATIVE_USB4_SUPPORT; + } + + kfree(context.ret.pointer); +} + +/* + * Native control of USB4 capabilities. If any of the tunneling bits is + * set it means OS is in control and we use software based connection + * manager. + */ +u32 osc_sb_native_usb4_control; +EXPORT_SYMBOL_GPL(osc_sb_native_usb4_control); + +static void acpi_bus_decode_usb_osc(const char *msg, u32 bits) +{ + printk(KERN_INFO PREFIX "%s USB3%c DisplayPort%c PCIe%c XDomain%c\n", msg, + (bits & OSC_USB_USB3_TUNNELING) ? '+' : '-', + (bits & OSC_USB_DP_TUNNELING) ? '+' : '-', + (bits & OSC_USB_PCIE_TUNNELING) ? '+' : '-', + (bits & OSC_USB_XDOMAIN) ? '+' : '-'); +} + +static u8 sb_usb_uuid_str[] = "23A0D13A-26AB-486C-9C5F-0FFA525A575A"; +static void acpi_bus_osc_negotiate_usb_control(void) +{ + u32 capbuf[3]; + struct acpi_osc_context context = { + .uuid_str = sb_usb_uuid_str, + .rev = 1, + .cap.length = sizeof(capbuf), + .cap.pointer = capbuf, + }; + acpi_handle handle; + acpi_status status; + u32 control; + + if (!osc_sb_native_usb4_support_confirmed) + return; + + if (ACPI_FAILURE(acpi_get_handle(NULL, "\\_SB", &handle))) + return; + + control = OSC_USB_USB3_TUNNELING | OSC_USB_DP_TUNNELING | + OSC_USB_PCIE_TUNNELING | OSC_USB_XDOMAIN; + + capbuf[OSC_QUERY_DWORD] = 0; + capbuf[OSC_SUPPORT_DWORD] = 0; + capbuf[OSC_CONTROL_DWORD] = control; + + status = acpi_run_osc(handle, &context); + if (ACPI_FAILURE(status)) + return; + + if (context.ret.length != sizeof(capbuf)) { + printk(KERN_INFO PREFIX "USB4 _OSC: returned invalid length buffer\n"); + goto out_free; } + osc_sb_native_usb4_control = + control & ((u32 *)context.ret.pointer)[OSC_CONTROL_DWORD]; + + acpi_bus_decode_usb_osc("USB4 _OSC: OS supports", control); + acpi_bus_decode_usb_osc("USB4 _OSC: OS controls", + osc_sb_native_usb4_control); + +out_free: kfree(context.ret.pointer); } @@ -1188,6 +1263,7 @@ static int __init acpi_bus_init(void) * so it must be run after ACPI_FULL_INITIALIZATION */ acpi_bus_osc_negotiate_platform_control(); + acpi_bus_osc_negotiate_usb_control(); /* * _PDC control method may load dynamic SSDT tables, diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 2630c2e953f7..ac68c2d4e393 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -546,9 +546,19 @@ acpi_status acpi_run_osc(acpi_handle handle, struct acpi_osc_context *context); #define OSC_SB_OSLPI_SUPPORT 0x00000100 #define OSC_SB_CPC_DIVERSE_HIGH_SUPPORT 0x00001000 #define OSC_SB_GENERIC_INITIATOR_SUPPORT 0x00002000 +#define OSC_SB_NATIVE_USB4_SUPPORT 0x00040000 extern bool osc_sb_apei_support_acked; extern bool osc_pc_lpi_support_confirmed; +extern bool osc_sb_native_usb4_support_confirmed; + +/* USB4 Capabilities */ +#define OSC_USB_USB3_TUNNELING 0x00000001 +#define OSC_USB_DP_TUNNELING 0x00000002 +#define OSC_USB_PCIE_TUNNELING 0x00000004 +#define OSC_USB_XDOMAIN 0x00000008 + +extern u32 osc_sb_native_usb4_control; /* PCI Host Bridge _OSC: Capabilities DWORD 2: Support Field */ #define OSC_PCI_EXT_CONFIG_SUPPORT 0x00000001 -- cgit v1.2.3 From 2c07ded06427dd3339278487a1413d5e478f05f9 Mon Sep 17 00:00:00 2001 From: Brijesh Singh Date: Mon, 4 Jan 2021 09:17:49 -0600 Subject: KVM/SVM: add support for SEV attestation command The SEV FW version >= 0.23 added a new command that can be used to query the attestation report containing the SHA-256 digest of the guest memory encrypted through the KVM_SEV_LAUNCH_UPDATE_{DATA, VMSA} commands and sign the report with the Platform Endorsement Key (PEK). See the SEV FW API spec section 6.8 for more details. Note there already exist a command (KVM_SEV_LAUNCH_MEASURE) that can be used to get the SHA-256 digest. The main difference between the KVM_SEV_LAUNCH_MEASURE and KVM_SEV_ATTESTATION_REPORT is that the latter can be called while the guest is running and the measurement value is signed with PEK. Cc: James Bottomley Cc: Tom Lendacky Cc: David Rientjes Cc: Paolo Bonzini Cc: Sean Christopherson Cc: Borislav Petkov Cc: John Allen Cc: Herbert Xu Cc: linux-crypto@vger.kernel.org Reviewed-by: Tom Lendacky Acked-by: David Rientjes Tested-by: James Bottomley Signed-off-by: Brijesh Singh Message-Id: <20210104151749.30248-1-brijesh.singh@amd.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/amd-memory-encryption.rst | 21 +++++++ arch/x86/kvm/svm/sev.c | 71 ++++++++++++++++++++++++ drivers/crypto/ccp/sev-dev.c | 1 + include/linux/psp-sev.h | 17 ++++++ include/uapi/linux/kvm.h | 8 +++ 5 files changed, 118 insertions(+) (limited to 'include/linux') diff --git a/Documentation/virt/kvm/amd-memory-encryption.rst b/Documentation/virt/kvm/amd-memory-encryption.rst index 09a8f2a34e39..469a6308765b 100644 --- a/Documentation/virt/kvm/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/amd-memory-encryption.rst @@ -263,6 +263,27 @@ Returns: 0 on success, -negative on error __u32 trans_len; }; +10. KVM_SEV_GET_ATTESTATION_REPORT +---------------------------------- + +The KVM_SEV_GET_ATTESTATION_REPORT command can be used by the hypervisor to query the attestation +report containing the SHA-256 digest of the guest memory and VMSA passed through the KVM_SEV_LAUNCH +commands and signed with the PEK. The digest returned by the command should match the digest +used by the guest owner with the KVM_SEV_LAUNCH_MEASURE. + +Parameters (in): struct kvm_sev_attestation + +Returns: 0 on success, -negative on error + +:: + + struct kvm_sev_attestation_report { + __u8 mnonce[16]; /* A random mnonce that will be placed in the report */ + + __u64 uaddr; /* userspace address where the report should be copied */ + __u32 len; + }; + References ========== diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 48017fef1cd9..8dfe8988be8d 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -1041,6 +1041,74 @@ e_unpin_memory: return ret; } +static int sev_get_attestation_report(struct kvm *kvm, struct kvm_sev_cmd *argp) +{ + void __user *report = (void __user *)(uintptr_t)argp->data; + struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; + struct sev_data_attestation_report *data; + struct kvm_sev_attestation_report params; + void __user *p; + void *blob = NULL; + int ret; + + if (!sev_guest(kvm)) + return -ENOTTY; + + if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) + return -EFAULT; + + data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT); + if (!data) + return -ENOMEM; + + /* User wants to query the blob length */ + if (!params.len) + goto cmd; + + p = (void __user *)(uintptr_t)params.uaddr; + if (p) { + if (params.len > SEV_FW_BLOB_MAX_SIZE) { + ret = -EINVAL; + goto e_free; + } + + ret = -ENOMEM; + blob = kmalloc(params.len, GFP_KERNEL); + if (!blob) + goto e_free; + + data->address = __psp_pa(blob); + data->len = params.len; + memcpy(data->mnonce, params.mnonce, sizeof(params.mnonce)); + } +cmd: + data->handle = sev->handle; + ret = sev_issue_cmd(kvm, SEV_CMD_ATTESTATION_REPORT, data, &argp->error); + /* + * If we query the session length, FW responded with expected data. + */ + if (!params.len) + goto done; + + if (ret) + goto e_free_blob; + + if (blob) { + if (copy_to_user(p, blob, params.len)) + ret = -EFAULT; + } + +done: + params.len = data->len; + if (copy_to_user(report, ¶ms, sizeof(params))) + ret = -EFAULT; +e_free_blob: + kfree(blob); +e_free: + kfree(data); + return ret; +} + int svm_mem_enc_op(struct kvm *kvm, void __user *argp) { struct kvm_sev_cmd sev_cmd; @@ -1091,6 +1159,9 @@ int svm_mem_enc_op(struct kvm *kvm, void __user *argp) case KVM_SEV_LAUNCH_SECRET: r = sev_launch_secret(kvm, &sev_cmd); break; + case KVM_SEV_GET_ATTESTATION_REPORT: + r = sev_get_attestation_report(kvm, &sev_cmd); + break; default: r = -EINVAL; goto out; diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 476113e12489..cb9b4c4e371e 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -128,6 +128,7 @@ static int sev_cmd_buffer_len(int cmd) case SEV_CMD_LAUNCH_UPDATE_SECRET: return sizeof(struct sev_data_launch_secret); case SEV_CMD_DOWNLOAD_FIRMWARE: return sizeof(struct sev_data_download_firmware); case SEV_CMD_GET_ID: return sizeof(struct sev_data_get_id); + case SEV_CMD_ATTESTATION_REPORT: return sizeof(struct sev_data_attestation_report); default: return 0; } diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index 49d155cd2dfe..b801ead1e2bb 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -66,6 +66,7 @@ enum sev_cmd { SEV_CMD_LAUNCH_MEASURE = 0x033, SEV_CMD_LAUNCH_UPDATE_SECRET = 0x034, SEV_CMD_LAUNCH_FINISH = 0x035, + SEV_CMD_ATTESTATION_REPORT = 0x036, /* Guest migration commands (outgoing) */ SEV_CMD_SEND_START = 0x040, @@ -483,6 +484,22 @@ struct sev_data_dbg { u32 len; /* In */ } __packed; +/** + * struct sev_data_attestation_report - SEV_ATTESTATION_REPORT command parameters + * + * @handle: handle of the VM + * @mnonce: a random nonce that will be included in the report. + * @address: physical address where the report will be copied. + * @len: length of the physical buffer. + */ +struct sev_data_attestation_report { + u32 handle; /* In */ + u32 reserved; + u64 address; /* In */ + u8 mnonce[16]; /* In */ + u32 len; /* In/Out */ +} __packed; + #ifdef CONFIG_CRYPTO_DEV_SP_PSP /** diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 374c67875cdb..07c194e2c302 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1593,6 +1593,8 @@ enum sev_cmd_id { KVM_SEV_DBG_ENCRYPT, /* Guest certificates commands */ KVM_SEV_CERT_EXPORT, + /* Attestation report */ + KVM_SEV_GET_ATTESTATION_REPORT, KVM_SEV_NR_MAX, }; @@ -1645,6 +1647,12 @@ struct kvm_sev_dbg { __u32 len; }; +struct kvm_sev_attestation_report { + __u8 mnonce[16]; + __u64 uaddr; + __u32 len; +}; + #define KVM_DEV_ASSIGN_ENABLE_IOMMU (1 << 0) #define KVM_DEV_ASSIGN_PCI_2_3 (1 << 1) #define KVM_DEV_ASSIGN_MASK_INTX (1 << 2) -- cgit v1.2.3 From 26128cb6c7e6731fe644c687af97733adfdb5ee9 Mon Sep 17 00:00:00 2001 From: Ben Gardon Date: Tue, 2 Feb 2021 10:57:12 -0800 Subject: locking/rwlocks: Add contention detection for rwlocks rwlocks do not currently have any facility to detect contention like spinlocks do. In order to allow users of rwlocks to better manage latency, add contention detection for queued rwlocks. CC: Ingo Molnar CC: Will Deacon Acked-by: Peter Zijlstra Acked-by: Davidlohr Bueso Acked-by: Waiman Long Acked-by: Paolo Bonzini Signed-off-by: Ben Gardon Message-Id: <20210202185734.1680553-7-bgardon@google.com> Signed-off-by: Paolo Bonzini --- include/asm-generic/qrwlock.h | 24 ++++++++++++++++++------ include/linux/rwlock.h | 7 +++++++ 2 files changed, 25 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/asm-generic/qrwlock.h b/include/asm-generic/qrwlock.h index 84ce841ce735..0020d3b820a7 100644 --- a/include/asm-generic/qrwlock.h +++ b/include/asm-generic/qrwlock.h @@ -14,6 +14,7 @@ #include #include +#include /* * Writer states & reader shift and bias. @@ -116,15 +117,26 @@ static inline void queued_write_unlock(struct qrwlock *lock) smp_store_release(&lock->wlocked, 0); } +/** + * queued_rwlock_is_contended - check if the lock is contended + * @lock : Pointer to queue rwlock structure + * Return: 1 if lock contended, 0 otherwise + */ +static inline int queued_rwlock_is_contended(struct qrwlock *lock) +{ + return arch_spin_is_locked(&lock->wait_lock); +} + /* * Remapping rwlock architecture specific functions to the corresponding * queue rwlock functions. */ -#define arch_read_lock(l) queued_read_lock(l) -#define arch_write_lock(l) queued_write_lock(l) -#define arch_read_trylock(l) queued_read_trylock(l) -#define arch_write_trylock(l) queued_write_trylock(l) -#define arch_read_unlock(l) queued_read_unlock(l) -#define arch_write_unlock(l) queued_write_unlock(l) +#define arch_read_lock(l) queued_read_lock(l) +#define arch_write_lock(l) queued_write_lock(l) +#define arch_read_trylock(l) queued_read_trylock(l) +#define arch_write_trylock(l) queued_write_trylock(l) +#define arch_read_unlock(l) queued_read_unlock(l) +#define arch_write_unlock(l) queued_write_unlock(l) +#define arch_rwlock_is_contended(l) queued_rwlock_is_contended(l) #endif /* __ASM_GENERIC_QRWLOCK_H */ diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h index 3dcd617e65ae..7ce9a51ae5c0 100644 --- a/include/linux/rwlock.h +++ b/include/linux/rwlock.h @@ -128,4 +128,11 @@ do { \ 1 : ({ local_irq_restore(flags); 0; }); \ }) +#ifdef arch_rwlock_is_contended +#define rwlock_is_contended(lock) \ + arch_rwlock_is_contended(&(lock)->raw_lock) +#else +#define rwlock_is_contended(lock) ((void)(lock), 0) +#endif /* arch_rwlock_is_contended */ + #endif /* __LINUX_RWLOCK_H */ -- cgit v1.2.3 From a09a689a534183c48f200bc2de1ae61ae9c462ad Mon Sep 17 00:00:00 2001 From: Ben Gardon Date: Tue, 2 Feb 2021 10:57:13 -0800 Subject: sched: Add needbreak for rwlocks Contention awareness while holding a spin lock is essential for reducing latency when long running kernel operations can hold that lock. Add the same contention detection interface for read/write spin locks. CC: Ingo Molnar CC: Will Deacon Acked-by: Peter Zijlstra Acked-by: Davidlohr Bueso Acked-by: Waiman Long Acked-by: Paolo Bonzini Signed-off-by: Ben Gardon Message-Id: <20210202185734.1680553-8-bgardon@google.com> Signed-off-by: Paolo Bonzini --- include/linux/sched.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index 6e3a5eeec509..5d1378e5a040 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1912,6 +1912,23 @@ static inline int spin_needbreak(spinlock_t *lock) #endif } +/* + * Check if a rwlock is contended. + * Returns non-zero if there is another task waiting on the rwlock. + * Returns zero if the lock is not contended or the system / underlying + * rwlock implementation does not support contention detection. + * Technically does not depend on CONFIG_PREEMPTION, but a general need + * for low latency. + */ +static inline int rwlock_needbreak(rwlock_t *lock) +{ +#ifdef CONFIG_PREEMPTION + return rwlock_is_contended(lock); +#else + return 0; +#endif +} + static __always_inline bool need_resched(void) { return unlikely(tif_need_resched()); -- cgit v1.2.3 From f3d4b4b1dc1c5fb9ea17cac14133463bfe72f170 Mon Sep 17 00:00:00 2001 From: Ben Gardon Date: Tue, 2 Feb 2021 10:57:14 -0800 Subject: sched: Add cond_resched_rwlock Safely rescheduling while holding a spin lock is essential for keeping long running kernel operations running smoothly. Add the facility to cond_resched rwlocks. CC: Ingo Molnar CC: Will Deacon Acked-by: Peter Zijlstra Acked-by: Davidlohr Bueso Acked-by: Waiman Long Acked-by: Paolo Bonzini Signed-off-by: Ben Gardon Message-Id: <20210202185734.1680553-9-bgardon@google.com> Signed-off-by: Paolo Bonzini --- include/linux/sched.h | 12 ++++++++++++ kernel/sched/core.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) (limited to 'include/linux') diff --git a/include/linux/sched.h b/include/linux/sched.h index 5d1378e5a040..3052d16da3cf 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1883,12 +1883,24 @@ static inline int _cond_resched(void) { return 0; } }) extern int __cond_resched_lock(spinlock_t *lock); +extern int __cond_resched_rwlock_read(rwlock_t *lock); +extern int __cond_resched_rwlock_write(rwlock_t *lock); #define cond_resched_lock(lock) ({ \ ___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\ __cond_resched_lock(lock); \ }) +#define cond_resched_rwlock_read(lock) ({ \ + __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \ + __cond_resched_rwlock_read(lock); \ +}) + +#define cond_resched_rwlock_write(lock) ({ \ + __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \ + __cond_resched_rwlock_write(lock); \ +}) + static inline void cond_resched_rcu(void) { #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 15d2562118d1..ade357642279 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6695,6 +6695,46 @@ int __cond_resched_lock(spinlock_t *lock) } EXPORT_SYMBOL(__cond_resched_lock); +int __cond_resched_rwlock_read(rwlock_t *lock) +{ + int resched = should_resched(PREEMPT_LOCK_OFFSET); + int ret = 0; + + lockdep_assert_held_read(lock); + + if (rwlock_needbreak(lock) || resched) { + read_unlock(lock); + if (resched) + preempt_schedule_common(); + else + cpu_relax(); + ret = 1; + read_lock(lock); + } + return ret; +} +EXPORT_SYMBOL(__cond_resched_rwlock_read); + +int __cond_resched_rwlock_write(rwlock_t *lock) +{ + int resched = should_resched(PREEMPT_LOCK_OFFSET); + int ret = 0; + + lockdep_assert_held_write(lock); + + if (rwlock_needbreak(lock) || resched) { + write_unlock(lock); + if (resched) + preempt_schedule_common(); + else + cpu_relax(); + ret = 1; + write_lock(lock); + } + return ret; +} +EXPORT_SYMBOL(__cond_resched_rwlock_write); + /** * yield - yield the current processor to other threads. * -- cgit v1.2.3 From 531810caa9f4bc99ffbb90e09256792c56a6b07a Mon Sep 17 00:00:00 2001 From: Ben Gardon Date: Tue, 2 Feb 2021 10:57:24 -0800 Subject: KVM: x86/mmu: Use an rwlock for the x86 MMU Add a read / write lock to be used in place of the MMU spinlock on x86. The rwlock will enable the TDP MMU to handle page faults, and other operations in parallel in future commits. Reviewed-by: Peter Feiner Signed-off-by: Ben Gardon Message-Id: <20210202185734.1680553-19-bgardon@google.com> [Introduce virt/kvm/mmu_lock.h - Paolo] Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 + arch/x86/kvm/mmu/mmu.c | 90 ++++++++++++++++++++--------------------- arch/x86/kvm/mmu/page_track.c | 8 ++-- arch/x86/kvm/mmu/paging_tmpl.h | 8 ++-- arch/x86/kvm/mmu/tdp_mmu.c | 20 ++++----- arch/x86/kvm/x86.c | 4 +- include/linux/kvm_host.h | 5 +++ virt/kvm/dirty_ring.c | 5 ++- virt/kvm/kvm_main.c | 37 +++++++++-------- virt/kvm/mmu_lock.h | 23 +++++++++++ 10 files changed, 118 insertions(+), 84 deletions(-) create mode 100644 virt/kvm/mmu_lock.h (limited to 'include/linux') diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index fa7b2df6422b..c445a51244d3 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -348,6 +348,8 @@ struct kvm_mmu_root_info { #define KVM_MMU_NUM_PREV_ROOTS 3 +#define KVM_HAVE_MMU_RWLOCK + struct kvm_mmu_page; /* diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 5b364ff9c115..329930d57774 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2010,9 +2010,9 @@ static void mmu_sync_children(struct kvm_vcpu *vcpu, flush |= kvm_sync_page(vcpu, sp, &invalid_list); mmu_pages_clear_parents(&parents); } - if (need_resched() || spin_needbreak(&vcpu->kvm->mmu_lock)) { + if (need_resched() || rwlock_needbreak(&vcpu->kvm->mmu_lock)) { kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); - cond_resched_lock(&vcpu->kvm->mmu_lock); + cond_resched_rwlock_write(&vcpu->kvm->mmu_lock); flush = false; } } @@ -2464,7 +2464,7 @@ static int make_mmu_pages_available(struct kvm_vcpu *vcpu) */ void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages) { - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) { kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages - @@ -2475,7 +2475,7 @@ void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages) kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages; - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); } int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) @@ -2486,7 +2486,7 @@ int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) pgprintk("%s: looking for gfn %llx\n", __func__, gfn); r = 0; - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); for_each_gfn_indirect_valid_sp(kvm, sp, gfn) { pgprintk("%s: gfn %llx role %x\n", __func__, gfn, sp->role.word); @@ -2494,7 +2494,7 @@ int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list); } kvm_mmu_commit_zap_page(kvm, &invalid_list); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); return r; } @@ -3186,7 +3186,7 @@ void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, return; } - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) @@ -3209,7 +3209,7 @@ void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, } kvm_mmu_commit_zap_page(kvm, &invalid_list); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); } EXPORT_SYMBOL_GPL(kvm_mmu_free_roots); @@ -3230,16 +3230,16 @@ static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, gva_t gva, { struct kvm_mmu_page *sp; - spin_lock(&vcpu->kvm->mmu_lock); + write_lock(&vcpu->kvm->mmu_lock); if (make_mmu_pages_available(vcpu)) { - spin_unlock(&vcpu->kvm->mmu_lock); + write_unlock(&vcpu->kvm->mmu_lock); return INVALID_PAGE; } sp = kvm_mmu_get_page(vcpu, gfn, gva, level, direct, ACC_ALL); ++sp->root_count; - spin_unlock(&vcpu->kvm->mmu_lock); + write_unlock(&vcpu->kvm->mmu_lock); return __pa(sp->spt); } @@ -3410,17 +3410,17 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) !smp_load_acquire(&sp->unsync_children)) return; - spin_lock(&vcpu->kvm->mmu_lock); + write_lock(&vcpu->kvm->mmu_lock); kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC); mmu_sync_children(vcpu, sp); kvm_mmu_audit(vcpu, AUDIT_POST_SYNC); - spin_unlock(&vcpu->kvm->mmu_lock); + write_unlock(&vcpu->kvm->mmu_lock); return; } - spin_lock(&vcpu->kvm->mmu_lock); + write_lock(&vcpu->kvm->mmu_lock); kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC); for (i = 0; i < 4; ++i) { @@ -3434,7 +3434,7 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) } kvm_mmu_audit(vcpu, AUDIT_POST_SYNC); - spin_unlock(&vcpu->kvm->mmu_lock); + write_unlock(&vcpu->kvm->mmu_lock); } EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots); @@ -3718,7 +3718,7 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, return r; r = RET_PF_RETRY; - spin_lock(&vcpu->kvm->mmu_lock); + write_lock(&vcpu->kvm->mmu_lock); if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) goto out_unlock; r = make_mmu_pages_available(vcpu); @@ -3733,7 +3733,7 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, prefault, is_tdp); out_unlock: - spin_unlock(&vcpu->kvm->mmu_lock); + write_unlock(&vcpu->kvm->mmu_lock); kvm_release_pfn_clean(pfn); return r; } @@ -4959,7 +4959,7 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, */ mmu_topup_memory_caches(vcpu, true); - spin_lock(&vcpu->kvm->mmu_lock); + write_lock(&vcpu->kvm->mmu_lock); gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes); @@ -4991,7 +4991,7 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, } kvm_mmu_flush_or_zap(vcpu, &invalid_list, remote_flush, local_flush); kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE); - spin_unlock(&vcpu->kvm->mmu_lock); + write_unlock(&vcpu->kvm->mmu_lock); } int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) @@ -5189,14 +5189,14 @@ slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot, if (iterator.rmap) flush |= fn(kvm, iterator.rmap); - if (need_resched() || spin_needbreak(&kvm->mmu_lock)) { + if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) { if (flush && lock_flush_tlb) { kvm_flush_remote_tlbs_with_address(kvm, start_gfn, iterator.gfn - start_gfn + 1); flush = false; } - cond_resched_lock(&kvm->mmu_lock); + cond_resched_rwlock_write(&kvm->mmu_lock); } } @@ -5346,7 +5346,7 @@ restart: * be in active use by the guest. */ if (batch >= BATCH_ZAP_PAGES && - cond_resched_lock(&kvm->mmu_lock)) { + cond_resched_rwlock_write(&kvm->mmu_lock)) { batch = 0; goto restart; } @@ -5379,7 +5379,7 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) { lockdep_assert_held(&kvm->slots_lock); - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); trace_kvm_mmu_zap_all_fast(kvm); /* @@ -5406,7 +5406,7 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) if (kvm->arch.tdp_mmu_enabled) kvm_tdp_mmu_zap_all(kvm); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); } static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm) @@ -5448,7 +5448,7 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) int i; bool flush; - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { slots = __kvm_memslots(kvm, i); kvm_for_each_memslot(memslot, slots) { @@ -5472,7 +5472,7 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) kvm_flush_remote_tlbs(kvm); } - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); } static bool slot_rmap_write_protect(struct kvm *kvm, @@ -5487,12 +5487,12 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, { bool flush; - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect, start_level, KVM_MAX_HUGEPAGE_LEVEL, false); if (kvm->arch.tdp_mmu_enabled) flush |= kvm_tdp_mmu_wrprot_slot(kvm, memslot, PG_LEVEL_4K); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); /* * We can flush all the TLBs out of the mmu lock without TLB @@ -5552,13 +5552,13 @@ void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, const struct kvm_memory_slot *memslot) { /* FIXME: const-ify all uses of struct kvm_memory_slot. */ - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot, kvm_mmu_zap_collapsible_spte, true); if (kvm->arch.tdp_mmu_enabled) kvm_tdp_mmu_zap_collapsible_sptes(kvm, memslot); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); } void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, @@ -5581,11 +5581,11 @@ void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, { bool flush; - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false); if (kvm->arch.tdp_mmu_enabled) flush |= kvm_tdp_mmu_clear_dirty_slot(kvm, memslot); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); /* * It's also safe to flush TLBs out of mmu lock here as currently this @@ -5603,12 +5603,12 @@ void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, { bool flush; - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect, false); if (kvm->arch.tdp_mmu_enabled) flush |= kvm_tdp_mmu_wrprot_slot(kvm, memslot, PG_LEVEL_2M); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); if (flush) kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); @@ -5620,11 +5620,11 @@ void kvm_mmu_slot_set_dirty(struct kvm *kvm, { bool flush; - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false); if (kvm->arch.tdp_mmu_enabled) flush |= kvm_tdp_mmu_slot_set_dirty(kvm, memslot); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); if (flush) kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); @@ -5637,14 +5637,14 @@ void kvm_mmu_zap_all(struct kvm *kvm) LIST_HEAD(invalid_list); int ign; - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); restart: list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) { if (WARN_ON(sp->role.invalid)) continue; if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign)) goto restart; - if (cond_resched_lock(&kvm->mmu_lock)) + if (cond_resched_rwlock_write(&kvm->mmu_lock)) goto restart; } @@ -5653,7 +5653,7 @@ restart: if (kvm->arch.tdp_mmu_enabled) kvm_tdp_mmu_zap_all(kvm); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); } void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) @@ -5713,7 +5713,7 @@ mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) continue; idx = srcu_read_lock(&kvm->srcu); - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); if (kvm_has_zapped_obsolete_pages(kvm)) { kvm_mmu_commit_zap_page(kvm, @@ -5724,7 +5724,7 @@ mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) freed = kvm_mmu_zap_oldest_mmu_pages(kvm, sc->nr_to_scan); unlock: - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); srcu_read_unlock(&kvm->srcu, idx); /* @@ -5944,7 +5944,7 @@ static void kvm_recover_nx_lpages(struct kvm *kvm) ulong to_zap; rcu_idx = srcu_read_lock(&kvm->srcu); - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); ratio = READ_ONCE(nx_huge_pages_recovery_ratio); to_zap = ratio ? DIV_ROUND_UP(kvm->stat.nx_lpage_splits, ratio) : 0; @@ -5969,14 +5969,14 @@ static void kvm_recover_nx_lpages(struct kvm *kvm) WARN_ON_ONCE(sp->lpage_disallowed); } - if (need_resched() || spin_needbreak(&kvm->mmu_lock)) { + if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) { kvm_mmu_commit_zap_page(kvm, &invalid_list); - cond_resched_lock(&kvm->mmu_lock); + cond_resched_rwlock_write(&kvm->mmu_lock); } } kvm_mmu_commit_zap_page(kvm, &invalid_list); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); srcu_read_unlock(&kvm->srcu, rcu_idx); } diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 8443a675715b..34bb0ec69bd8 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -184,9 +184,9 @@ kvm_page_track_register_notifier(struct kvm *kvm, head = &kvm->arch.track_notifier_head; - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); hlist_add_head_rcu(&n->node, &head->track_notifier_list); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); } EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier); @@ -202,9 +202,9 @@ kvm_page_track_unregister_notifier(struct kvm *kvm, head = &kvm->arch.track_notifier_head; - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); hlist_del_rcu(&n->node); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); synchronize_srcu(&head->track_srcu); } EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier); diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 50e268eb8e1a..d9f66cc459e8 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -868,7 +868,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, } r = RET_PF_RETRY; - spin_lock(&vcpu->kvm->mmu_lock); + write_lock(&vcpu->kvm->mmu_lock); if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) goto out_unlock; @@ -881,7 +881,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, kvm_mmu_audit(vcpu, AUDIT_POST_PAGE_FAULT); out_unlock: - spin_unlock(&vcpu->kvm->mmu_lock); + write_unlock(&vcpu->kvm->mmu_lock); kvm_release_pfn_clean(pfn); return r; } @@ -919,7 +919,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa) return; } - spin_lock(&vcpu->kvm->mmu_lock); + write_lock(&vcpu->kvm->mmu_lock); for_each_shadow_entry_using_root(vcpu, root_hpa, gva, iterator) { level = iterator.level; sptep = iterator.sptep; @@ -954,7 +954,7 @@ static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa) if (!is_shadow_present_pte(*sptep) || !sp->unsync_children) break; } - spin_unlock(&vcpu->kvm->mmu_lock); + write_unlock(&vcpu->kvm->mmu_lock); } /* Note, @addr is a GPA when gva_to_gpa() translates an L2 GPA to an L1 GPA. */ diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 9e4009068920..f1fbed72e149 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -59,7 +59,7 @@ static void tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root) static inline bool tdp_mmu_next_root_valid(struct kvm *kvm, struct kvm_mmu_page *root) { - lockdep_assert_held(&kvm->mmu_lock); + lockdep_assert_held_write(&kvm->mmu_lock); if (list_entry_is_head(root, &kvm->arch.tdp_mmu_roots, link)) return false; @@ -117,7 +117,7 @@ void kvm_tdp_mmu_free_root(struct kvm *kvm, struct kvm_mmu_page *root) { gfn_t max_gfn = 1ULL << (shadow_phys_bits - PAGE_SHIFT); - lockdep_assert_held(&kvm->mmu_lock); + lockdep_assert_held_write(&kvm->mmu_lock); WARN_ON(root->root_count); WARN_ON(!root->tdp_mmu_page); @@ -170,13 +170,13 @@ static struct kvm_mmu_page *get_tdp_mmu_vcpu_root(struct kvm_vcpu *vcpu) role = page_role_for_level(vcpu, vcpu->arch.mmu->shadow_root_level); - spin_lock(&kvm->mmu_lock); + write_lock(&kvm->mmu_lock); /* Check for an existing root before allocating a new one. */ for_each_tdp_mmu_root(kvm, root) { if (root->role.word == role.word) { kvm_mmu_get_root(kvm, root); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); return root; } } @@ -186,7 +186,7 @@ static struct kvm_mmu_page *get_tdp_mmu_vcpu_root(struct kvm_vcpu *vcpu) list_add(&root->link, &kvm->arch.tdp_mmu_roots); - spin_unlock(&kvm->mmu_lock); + write_unlock(&kvm->mmu_lock); return root; } @@ -421,7 +421,7 @@ static inline void __tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter, struct kvm_mmu_page *root = sptep_to_sp(root_pt); int as_id = kvm_mmu_page_as_id(root); - lockdep_assert_held(&kvm->mmu_lock); + lockdep_assert_held_write(&kvm->mmu_lock); WRITE_ONCE(*rcu_dereference(iter->sptep), new_spte); @@ -492,13 +492,13 @@ static inline bool tdp_mmu_iter_cond_resched(struct kvm *kvm, if (iter->next_last_level_gfn == iter->yielded_gfn) return false; - if (need_resched() || spin_needbreak(&kvm->mmu_lock)) { + if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) { rcu_read_unlock(); if (flush) kvm_flush_remote_tlbs(kvm); - cond_resched_lock(&kvm->mmu_lock); + cond_resched_rwlock_write(&kvm->mmu_lock); rcu_read_lock(); WARN_ON(iter->gfn > iter->next_last_level_gfn); @@ -1103,7 +1103,7 @@ void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm, struct kvm_mmu_page *root; int root_as_id; - lockdep_assert_held(&kvm->mmu_lock); + lockdep_assert_held_write(&kvm->mmu_lock); for_each_tdp_mmu_root(kvm, root) { root_as_id = kvm_mmu_page_as_id(root); if (root_as_id != slot->as_id) @@ -1268,7 +1268,7 @@ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, int root_as_id; bool spte_set = false; - lockdep_assert_held(&kvm->mmu_lock); + lockdep_assert_held_write(&kvm->mmu_lock); for_each_tdp_mmu_root(kvm, root) { root_as_id = kvm_mmu_page_as_id(root); if (root_as_id != slot->as_id) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 912fc418ce99..b3b1cce939ab 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7112,9 +7112,9 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, if (vcpu->arch.mmu->direct_map) { unsigned int indirect_shadow_pages; - spin_lock(&vcpu->kvm->mmu_lock); + write_lock(&vcpu->kvm->mmu_lock); indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages; - spin_unlock(&vcpu->kvm->mmu_lock); + write_unlock(&vcpu->kvm->mmu_lock); if (indirect_shadow_pages) kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa)); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index f3b1013fb22c..f417447129b9 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -451,7 +451,12 @@ struct kvm_memslots { }; struct kvm { +#ifdef KVM_HAVE_MMU_RWLOCK + rwlock_t mmu_lock; +#else spinlock_t mmu_lock; +#endif /* KVM_HAVE_MMU_RWLOCK */ + struct mutex slots_lock; struct mm_struct *mm; /* userspace tied to this vm */ struct kvm_memslots __rcu *memslots[KVM_ADDRESS_SPACE_NUM]; diff --git a/virt/kvm/dirty_ring.c b/virt/kvm/dirty_ring.c index 790f17325f8d..7aafefc50aa7 100644 --- a/virt/kvm/dirty_ring.c +++ b/virt/kvm/dirty_ring.c @@ -9,6 +9,7 @@ #include #include #include +#include "mmu_lock.h" int __weak kvm_cpu_dirty_log_size(void) { @@ -60,9 +61,9 @@ static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask) if (!memslot || (offset + __fls(mask)) >= memslot->npages) return; - spin_lock(&kvm->mmu_lock); + KVM_MMU_LOCK(kvm); kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, mask); - spin_unlock(&kvm->mmu_lock); + KVM_MMU_UNLOCK(kvm); } int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 335a1a2b8edc..48ccdf4e3d04 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -58,6 +58,7 @@ #include "coalesced_mmio.h" #include "async_pf.h" +#include "mmu_lock.h" #include "vfio.h" #define CREATE_TRACE_POINTS @@ -459,13 +460,15 @@ static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, int idx; idx = srcu_read_lock(&kvm->srcu); - spin_lock(&kvm->mmu_lock); + + KVM_MMU_LOCK(kvm); + kvm->mmu_notifier_seq++; if (kvm_set_spte_hva(kvm, address, pte)) kvm_flush_remote_tlbs(kvm); - spin_unlock(&kvm->mmu_lock); + KVM_MMU_UNLOCK(kvm); srcu_read_unlock(&kvm->srcu, idx); } @@ -476,7 +479,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, int need_tlb_flush = 0, idx; idx = srcu_read_lock(&kvm->srcu); - spin_lock(&kvm->mmu_lock); + KVM_MMU_LOCK(kvm); /* * The count increase must become visible at unlock time as no * spte can be established without taking the mmu_lock and @@ -489,7 +492,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, if (need_tlb_flush || kvm->tlbs_dirty) kvm_flush_remote_tlbs(kvm); - spin_unlock(&kvm->mmu_lock); + KVM_MMU_UNLOCK(kvm); srcu_read_unlock(&kvm->srcu, idx); return 0; @@ -500,7 +503,7 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, { struct kvm *kvm = mmu_notifier_to_kvm(mn); - spin_lock(&kvm->mmu_lock); + KVM_MMU_LOCK(kvm); /* * This sequence increase will notify the kvm page fault that * the page that is going to be mapped in the spte could have @@ -514,7 +517,7 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, * in conjunction with the smp_rmb in mmu_notifier_retry(). */ kvm->mmu_notifier_count--; - spin_unlock(&kvm->mmu_lock); + KVM_MMU_UNLOCK(kvm); BUG_ON(kvm->mmu_notifier_count < 0); } @@ -528,13 +531,13 @@ static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn, int young, idx; idx = srcu_read_lock(&kvm->srcu); - spin_lock(&kvm->mmu_lock); + KVM_MMU_LOCK(kvm); young = kvm_age_hva(kvm, start, end); if (young) kvm_flush_remote_tlbs(kvm); - spin_unlock(&kvm->mmu_lock); + KVM_MMU_UNLOCK(kvm); srcu_read_unlock(&kvm->srcu, idx); return young; @@ -549,7 +552,7 @@ static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, int young, idx; idx = srcu_read_lock(&kvm->srcu); - spin_lock(&kvm->mmu_lock); + KVM_MMU_LOCK(kvm); /* * Even though we do not flush TLB, this will still adversely * affect performance on pre-Haswell Intel EPT, where there is @@ -564,7 +567,7 @@ static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn, * more sophisticated heuristic later. */ young = kvm_age_hva(kvm, start, end); - spin_unlock(&kvm->mmu_lock); + KVM_MMU_UNLOCK(kvm); srcu_read_unlock(&kvm->srcu, idx); return young; @@ -578,9 +581,9 @@ static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn, int young, idx; idx = srcu_read_lock(&kvm->srcu); - spin_lock(&kvm->mmu_lock); + KVM_MMU_LOCK(kvm); young = kvm_test_age_hva(kvm, address); - spin_unlock(&kvm->mmu_lock); + KVM_MMU_UNLOCK(kvm); srcu_read_unlock(&kvm->srcu, idx); return young; @@ -745,7 +748,7 @@ static struct kvm *kvm_create_vm(unsigned long type) if (!kvm) return ERR_PTR(-ENOMEM); - spin_lock_init(&kvm->mmu_lock); + KVM_MMU_LOCK_INIT(kvm); mmgrab(current->mm); kvm->mm = current->mm; kvm_eventfd_init(kvm); @@ -1525,7 +1528,7 @@ static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot); memset(dirty_bitmap_buffer, 0, n); - spin_lock(&kvm->mmu_lock); + KVM_MMU_LOCK(kvm); for (i = 0; i < n / sizeof(long); i++) { unsigned long mask; gfn_t offset; @@ -1541,7 +1544,7 @@ static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log) kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot, offset, mask); } - spin_unlock(&kvm->mmu_lock); + KVM_MMU_UNLOCK(kvm); } if (flush) @@ -1636,7 +1639,7 @@ static int kvm_clear_dirty_log_protect(struct kvm *kvm, if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n)) return -EFAULT; - spin_lock(&kvm->mmu_lock); + KVM_MMU_LOCK(kvm); for (offset = log->first_page, i = offset / BITS_PER_LONG, n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--; i++, offset += BITS_PER_LONG) { @@ -1659,7 +1662,7 @@ static int kvm_clear_dirty_log_protect(struct kvm *kvm, offset, mask); } } - spin_unlock(&kvm->mmu_lock); + KVM_MMU_UNLOCK(kvm); if (flush) kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); diff --git a/virt/kvm/mmu_lock.h b/virt/kvm/mmu_lock.h new file mode 100644 index 000000000000..9e1308f9734c --- /dev/null +++ b/virt/kvm/mmu_lock.h @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#ifndef KVM_MMU_LOCK_H +#define KVM_MMU_LOCK_H 1 + +/* + * Architectures can choose whether to use an rwlock or spinlock + * for the mmu_lock. These macros, for use in common code + * only, avoids using #ifdefs in places that must deal with + * multiple architectures. + */ + +#ifdef KVM_HAVE_MMU_RWLOCK +#define KVM_MMU_LOCK_INIT(kvm) rwlock_init(&(kvm)->mmu_lock) +#define KVM_MMU_LOCK(kvm) write_lock(&(kvm)->mmu_lock) +#define KVM_MMU_UNLOCK(kvm) write_unlock(&(kvm)->mmu_lock) +#else +#define KVM_MMU_LOCK_INIT(kvm) spin_lock_init(&(kvm)->mmu_lock) +#define KVM_MMU_LOCK(kvm) spin_lock(&(kvm)->mmu_lock) +#define KVM_MMU_UNLOCK(kvm) spin_unlock(&(kvm)->mmu_lock) +#endif /* KVM_HAVE_MMU_RWLOCK */ + +#endif -- cgit v1.2.3 From 7d8658ef65a4f891d0cff6340fa717b378384642 Mon Sep 17 00:00:00 2001 From: Saravana Kannan Date: Thu, 4 Feb 2021 16:14:22 +0800 Subject: OPP: Add function to look up required OPP's for a given OPP Add a function that allows looking up required OPPs given a source OPP table, destination OPP table and the source OPP. Signed-off-by: Saravana Kannan Signed-off-by: Hsin-Yi Wang [ Viresh: Rearranged code, fixed return errors ] Signed-off-by: Viresh Kumar --- drivers/opp/core.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/pm_opp.h | 7 +++++++ 2 files changed, 62 insertions(+) (limited to 'include/linux') diff --git a/drivers/opp/core.c b/drivers/opp/core.c index dc95d29e94c1..c3f3d9249cc5 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2398,6 +2398,61 @@ devm_pm_opp_attach_genpd(struct device *dev, const char **names, } EXPORT_SYMBOL_GPL(devm_pm_opp_attach_genpd); +/** + * dev_pm_opp_xlate_required_opp() - Find required OPP for @src_table OPP. + * @src_table: OPP table which has @dst_table as one of its required OPP table. + * @dst_table: Required OPP table of the @src_table. + * @src_opp: OPP from the @src_table. + * + * This function returns the OPP (present in @dst_table) pointed out by the + * "required-opps" property of the @src_opp (present in @src_table). + * + * The callers are required to call dev_pm_opp_put() for the returned OPP after + * use. + * + * Return: pointer to 'struct dev_pm_opp' on success and errorno otherwise. + */ +struct dev_pm_opp *dev_pm_opp_xlate_required_opp(struct opp_table *src_table, + struct opp_table *dst_table, + struct dev_pm_opp *src_opp) +{ + struct dev_pm_opp *opp, *dest_opp = ERR_PTR(-ENODEV); + int i; + + if (!src_table || !dst_table || !src_opp || + !src_table->required_opp_tables) + return ERR_PTR(-EINVAL); + + /* required-opps not fully initialized yet */ + if (lazy_linking_pending(src_table)) + return ERR_PTR(-EBUSY); + + for (i = 0; i < src_table->required_opp_count; i++) { + if (src_table->required_opp_tables[i] == dst_table) { + mutex_lock(&src_table->lock); + + list_for_each_entry(opp, &src_table->opp_list, node) { + if (opp == src_opp) { + dest_opp = opp->required_opps[i]; + dev_pm_opp_get(dest_opp); + break; + } + } + + mutex_unlock(&src_table->lock); + break; + } + } + + if (IS_ERR(dest_opp)) { + pr_err("%s: Couldn't find matching OPP (%p: %p)\n", __func__, + src_table, dst_table); + } + + return dest_opp; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_xlate_required_opp); + /** * dev_pm_opp_xlate_performance_state() - Find required OPP's pstate for src_table. * @src_table: OPP table which has dst_table as one of its required OPP table. diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index ab1d15ce559d..c0371efa4a0f 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -156,6 +156,7 @@ struct opp_table *devm_pm_opp_register_set_opp_helper(struct device *dev, int (* struct opp_table *dev_pm_opp_attach_genpd(struct device *dev, const char **names, struct device ***virt_devs); void dev_pm_opp_detach_genpd(struct opp_table *opp_table); struct opp_table *devm_pm_opp_attach_genpd(struct device *dev, const char **names, struct device ***virt_devs); +struct dev_pm_opp *dev_pm_opp_xlate_required_opp(struct opp_table *src_table, struct opp_table *dst_table, struct dev_pm_opp *src_opp); int dev_pm_opp_xlate_performance_state(struct opp_table *src_table, struct opp_table *dst_table, unsigned int pstate); int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq); int dev_pm_opp_set_opp(struct device *dev, struct dev_pm_opp *opp); @@ -367,6 +368,12 @@ static inline struct opp_table *devm_pm_opp_attach_genpd(struct device *dev, return ERR_PTR(-EOPNOTSUPP); } +static inline struct dev_pm_opp *dev_pm_opp_xlate_required_opp(struct opp_table *src_table, + struct opp_table *dst_table, struct dev_pm_opp *src_opp) +{ + return ERR_PTR(-EOPNOTSUPP); +} + static inline int dev_pm_opp_xlate_performance_state(struct opp_table *src_table, struct opp_table *dst_table, unsigned int pstate) { return -EOPNOTSUPP; -- cgit v1.2.3 From 26f9c7cc42a6dc036edf871544fd0e6b3a0601c1 Mon Sep 17 00:00:00 2001 From: Saravana Kannan Date: Thu, 4 Feb 2021 16:14:23 +0800 Subject: PM / devfreq: Cache OPP table reference in devfreq The OPP table can be used often in devfreq. Trying to get it each time can be expensive, so cache it in the devfreq struct. Signed-off-by: Saravana Kannan Acked-by: MyungJoo Ham Acked-by: Chanwoo Choi Signed-off-by: Hsin-Yi Wang [ Viresh: Added a blank line ] Signed-off-by: Viresh Kumar --- drivers/devfreq/devfreq.c | 7 +++++++ include/linux/devfreq.h | 2 ++ 2 files changed, 9 insertions(+) (limited to 'include/linux') diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index 6aa10de792b3..cefe84a10824 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -757,6 +757,9 @@ static void devfreq_dev_release(struct device *dev) if (devfreq->profile->exit) devfreq->profile->exit(devfreq->dev.parent); + if (devfreq->opp_table) + dev_pm_opp_put_opp_table(devfreq->opp_table); + mutex_destroy(&devfreq->lock); kfree(devfreq); } @@ -844,6 +847,10 @@ struct devfreq *devfreq_add_device(struct device *dev, } devfreq->suspend_freq = dev_pm_opp_get_suspend_opp_freq(dev); + devfreq->opp_table = dev_pm_opp_get_opp_table(dev); + if (IS_ERR(devfreq->opp_table)) + devfreq->opp_table = NULL; + atomic_set(&devfreq->suspend_count, 0); dev_set_name(&devfreq->dev, "%s", dev_name(dev)); diff --git a/include/linux/devfreq.h b/include/linux/devfreq.h index b6d3bae1c74d..26ea0850be9b 100644 --- a/include/linux/devfreq.h +++ b/include/linux/devfreq.h @@ -137,6 +137,7 @@ struct devfreq_stats { * using devfreq. * @profile: device-specific devfreq profile * @governor: method how to choose frequency based on the usage. + * @opp_table: Reference to OPP table of dev.parent, if one exists. * @nb: notifier block used to notify devfreq object that it should * reevaluate operable frequencies. Devfreq users may use * devfreq.nb to the corresponding register notifier call chain. @@ -173,6 +174,7 @@ struct devfreq { struct device dev; struct devfreq_dev_profile *profile; const struct devfreq_governor *governor; + struct opp_table *opp_table; struct notifier_block nb; struct delayed_work work; -- cgit v1.2.3 From b64afd949ee3a61e180813859b50aced26023c55 Mon Sep 17 00:00:00 2001 From: Prashant Malani Date: Tue, 2 Feb 2021 18:15:37 -0800 Subject: platform/chrome: cros_ec: Import Type C control command This command is used to communicate with the Chrome Embedded Controller (EC) regarding USB Type C events and state. These header updates are included in the latest Chrome OS EC headers [1] [1] https://chromium.googlesource.com/chromiumos/platform/ec/+/refs/heads/main/include/ec_commands.h Signed-off-by: Prashant Malani Link: https://lore.kernel.org/r/20210203021539.745239-1-pmalani@chromium.org Signed-off-by: Benson Leung --- include/linux/platform_data/cros_ec_commands.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'include/linux') diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h index 9787715540c7..50e1caad81be 100644 --- a/include/linux/platform_data/cros_ec_commands.h +++ b/include/linux/platform_data/cros_ec_commands.h @@ -5577,6 +5577,32 @@ struct ec_response_typec_discovery { struct svid_mode_info svids[0]; } __ec_align1; +/* USB Type-C commands for AP-controlled device policy. */ +#define EC_CMD_TYPEC_CONTROL 0x0132 + +enum typec_control_command { + TYPEC_CONTROL_COMMAND_EXIT_MODES, + TYPEC_CONTROL_COMMAND_CLEAR_EVENTS, + TYPEC_CONTROL_COMMAND_ENTER_MODE, +}; + +struct ec_params_typec_control { + uint8_t port; + uint8_t command; /* enum typec_control_command */ + uint16_t reserved; + + /* + * This section will be interpreted based on |command|. Define a + * placeholder structure to avoid having to increase the size and bump + * the command version when adding new sub-commands. + */ + union { + uint32_t clear_events_mask; + uint8_t mode_to_enter; /* enum typec_mode */ + uint8_t placeholder[128]; + }; +} __ec_align1; + /* * Gather all status information for a port. * -- cgit v1.2.3 From 5ae4a4b45d4396aa7f7c008c4ae9eca981d43f8c Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 2 Feb 2021 10:25:11 +0530 Subject: cpufreq: Remove CPUFREQ_STICKY flag During cpufreq driver's registration, if the ->init() callback for all the CPUs fail then there is not much point in keeping the driver around as it will only account for more of unnecessary noise, for example cpufreq core will try to suspend/resume the driver which never got registered properly. The removal of such a driver is avoided if the driver carries the CPUFREQ_STICKY flag. This was added way back [1] in 2004 and perhaps no one should ever need it now. A lot of drivers do set this flag, probably because they just copied it from other drivers. This was added earlier for some platforms [2] because their cpufreq drivers were getting registered before the CPUs were registered with subsys framework. And hence they used to fail. The same isn't true anymore though. The current code flow in the kernel is: start_kernel() -> kernel_init() -> kernel_init_freeable() -> do_basic_setup() -> driver_init() -> cpu_dev_init() -> subsys_system_register() //For CPUs -> do_initcalls() -> cpufreq_register_driver() Clearly, the CPUs will always get registered with subsys framework before any cpufreq driver can get probed. Remove the flag and update the relevant drivers. Link: https://git.kernel.org/pub/scm/linux/kernel/git/tglx/history.git/commit/include/linux/cpufreq.h?id=7cc9f0d9a1ab04cedc60d64fd8dcf7df224a3b4d # [1] Link: https://git.kernel.org/pub/scm/linux/kernel/git/tglx/history.git/commit/arch/arm/mach-sa1100/cpu-sa1100.c?id=f59d3bbe35f6268d729f51be82af8325d62f20f5 # [2] Signed-off-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq-dt.c | 2 +- drivers/cpufreq/cpufreq.c | 3 +-- drivers/cpufreq/davinci-cpufreq.c | 2 +- drivers/cpufreq/loongson1-cpufreq.c | 2 +- drivers/cpufreq/mediatek-cpufreq.c | 2 +- drivers/cpufreq/omap-cpufreq.c | 2 +- drivers/cpufreq/qcom-cpufreq-hw.c | 2 +- drivers/cpufreq/s3c24xx-cpufreq.c | 2 +- drivers/cpufreq/s5pv210-cpufreq.c | 2 +- drivers/cpufreq/sa1100-cpufreq.c | 2 +- drivers/cpufreq/sa1110-cpufreq.c | 2 +- drivers/cpufreq/scmi-cpufreq.c | 2 +- drivers/cpufreq/scpi-cpufreq.c | 2 +- drivers/cpufreq/spear-cpufreq.c | 2 +- drivers/cpufreq/tegra186-cpufreq.c | 2 +- drivers/cpufreq/tegra194-cpufreq.c | 3 +-- drivers/cpufreq/vexpress-spc-cpufreq.c | 3 +-- include/linux/cpufreq.h | 17 +++++++---------- 18 files changed, 24 insertions(+), 30 deletions(-) (limited to 'include/linux') diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index ad4234518ef6..b1e1bdc63b01 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -175,7 +175,7 @@ static int cpufreq_exit(struct cpufreq_policy *policy) } static struct cpufreq_driver dt_cpufreq_driver = { - .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK | + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV, .verify = cpufreq_generic_frequency_table_verify, .target_index = set_target, diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index d0a3525ce27f..7d0ae968def7 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2810,8 +2810,7 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) if (ret) goto err_boost_unreg; - if (!(cpufreq_driver->flags & CPUFREQ_STICKY) && - list_empty(&cpufreq_policy_list)) { + if (unlikely(list_empty(&cpufreq_policy_list))) { /* if all ->init() calls failed, unregister */ ret = -ENODEV; pr_debug("%s: No CPU initialized for driver %s\n", __func__, diff --git a/drivers/cpufreq/davinci-cpufreq.c b/drivers/cpufreq/davinci-cpufreq.c index 91f477a6cbc4..9e97f60f8199 100644 --- a/drivers/cpufreq/davinci-cpufreq.c +++ b/drivers/cpufreq/davinci-cpufreq.c @@ -95,7 +95,7 @@ static int davinci_cpu_init(struct cpufreq_policy *policy) } static struct cpufreq_driver davinci_driver = { - .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, .verify = cpufreq_generic_frequency_table_verify, .target_index = davinci_target, .get = cpufreq_generic_get, diff --git a/drivers/cpufreq/loongson1-cpufreq.c b/drivers/cpufreq/loongson1-cpufreq.c index 86f612593e49..fb72d709db56 100644 --- a/drivers/cpufreq/loongson1-cpufreq.c +++ b/drivers/cpufreq/loongson1-cpufreq.c @@ -116,7 +116,7 @@ static int ls1x_cpufreq_exit(struct cpufreq_policy *policy) static struct cpufreq_driver ls1x_cpufreq_driver = { .name = "cpufreq-ls1x", - .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, .verify = cpufreq_generic_frequency_table_verify, .target_index = ls1x_cpufreq_target, .get = cpufreq_generic_get, diff --git a/drivers/cpufreq/mediatek-cpufreq.c b/drivers/cpufreq/mediatek-cpufreq.c index 022e3e966e71..f2e491b25b07 100644 --- a/drivers/cpufreq/mediatek-cpufreq.c +++ b/drivers/cpufreq/mediatek-cpufreq.c @@ -463,7 +463,7 @@ static int mtk_cpufreq_exit(struct cpufreq_policy *policy) } static struct cpufreq_driver mtk_cpufreq_driver = { - .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK | + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_HAVE_GOVERNOR_PER_POLICY | CPUFREQ_IS_COOLING_DEV, .verify = cpufreq_generic_frequency_table_verify, diff --git a/drivers/cpufreq/omap-cpufreq.c b/drivers/cpufreq/omap-cpufreq.c index 3694bb030df3..e035ee216b0f 100644 --- a/drivers/cpufreq/omap-cpufreq.c +++ b/drivers/cpufreq/omap-cpufreq.c @@ -144,7 +144,7 @@ static int omap_cpu_exit(struct cpufreq_policy *policy) } static struct cpufreq_driver omap_driver = { - .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, .verify = cpufreq_generic_frequency_table_verify, .target_index = omap_target, .get = cpufreq_generic_get, diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 9ed5341dc515..2a3b4f44488b 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -374,7 +374,7 @@ static struct freq_attr *qcom_cpufreq_hw_attr[] = { }; static struct cpufreq_driver cpufreq_qcom_hw_driver = { - .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK | + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_HAVE_GOVERNOR_PER_POLICY | CPUFREQ_IS_COOLING_DEV, .verify = cpufreq_generic_frequency_table_verify, diff --git a/drivers/cpufreq/s3c24xx-cpufreq.c b/drivers/cpufreq/s3c24xx-cpufreq.c index 37efc0dc3f91..7380c32b238e 100644 --- a/drivers/cpufreq/s3c24xx-cpufreq.c +++ b/drivers/cpufreq/s3c24xx-cpufreq.c @@ -420,7 +420,7 @@ static int s3c_cpufreq_resume(struct cpufreq_policy *policy) #endif static struct cpufreq_driver s3c24xx_driver = { - .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, .target = s3c_cpufreq_target, .get = cpufreq_generic_get, .init = s3c_cpufreq_init, diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c index bed496cf8d24..69786e5bbf05 100644 --- a/drivers/cpufreq/s5pv210-cpufreq.c +++ b/drivers/cpufreq/s5pv210-cpufreq.c @@ -574,7 +574,7 @@ static int s5pv210_cpufreq_reboot_notifier_event(struct notifier_block *this, } static struct cpufreq_driver s5pv210_driver = { - .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, .verify = cpufreq_generic_frequency_table_verify, .target_index = s5pv210_target, .get = cpufreq_generic_get, diff --git a/drivers/cpufreq/sa1100-cpufreq.c b/drivers/cpufreq/sa1100-cpufreq.c index 5c075ef6adc0..252b9fc26124 100644 --- a/drivers/cpufreq/sa1100-cpufreq.c +++ b/drivers/cpufreq/sa1100-cpufreq.c @@ -186,7 +186,7 @@ static int __init sa1100_cpu_init(struct cpufreq_policy *policy) } static struct cpufreq_driver sa1100_driver __refdata = { - .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK | + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING, .verify = cpufreq_generic_frequency_table_verify, .target_index = sa1100_target, diff --git a/drivers/cpufreq/sa1110-cpufreq.c b/drivers/cpufreq/sa1110-cpufreq.c index d9d04d935b3a..1a83c8678a63 100644 --- a/drivers/cpufreq/sa1110-cpufreq.c +++ b/drivers/cpufreq/sa1110-cpufreq.c @@ -310,7 +310,7 @@ static int __init sa1110_cpu_init(struct cpufreq_policy *policy) /* sa1110_driver needs __refdata because it must remain after init registers * it with cpufreq_register_driver() */ static struct cpufreq_driver sa1110_driver __refdata = { - .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK | + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING, .verify = cpufreq_generic_frequency_table_verify, .target_index = sa1110_target, diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index 491a0a24fb1e..5bd03b59887f 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -217,7 +217,7 @@ static int scmi_cpufreq_exit(struct cpufreq_policy *policy) static struct cpufreq_driver scmi_cpufreq_driver = { .name = "scmi", - .flags = CPUFREQ_STICKY | CPUFREQ_HAVE_GOVERNOR_PER_POLICY | + .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV, .verify = cpufreq_generic_frequency_table_verify, diff --git a/drivers/cpufreq/scpi-cpufreq.c b/drivers/cpufreq/scpi-cpufreq.c index e5140ad63db8..d6a698a1b5d1 100644 --- a/drivers/cpufreq/scpi-cpufreq.c +++ b/drivers/cpufreq/scpi-cpufreq.c @@ -191,7 +191,7 @@ static int scpi_cpufreq_exit(struct cpufreq_policy *policy) static struct cpufreq_driver scpi_cpufreq_driver = { .name = "scpi-cpufreq", - .flags = CPUFREQ_STICKY | CPUFREQ_HAVE_GOVERNOR_PER_POLICY | + .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV, .verify = cpufreq_generic_frequency_table_verify, diff --git a/drivers/cpufreq/spear-cpufreq.c b/drivers/cpufreq/spear-cpufreq.c index 73bd8dc47074..7d0d62a06bf3 100644 --- a/drivers/cpufreq/spear-cpufreq.c +++ b/drivers/cpufreq/spear-cpufreq.c @@ -160,7 +160,7 @@ static int spear_cpufreq_init(struct cpufreq_policy *policy) static struct cpufreq_driver spear_cpufreq_driver = { .name = "cpufreq-spear", - .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, .verify = cpufreq_generic_frequency_table_verify, .target_index = spear_cpufreq_target, .get = cpufreq_generic_get, diff --git a/drivers/cpufreq/tegra186-cpufreq.c b/drivers/cpufreq/tegra186-cpufreq.c index e566ea298b59..5d1943e787b0 100644 --- a/drivers/cpufreq/tegra186-cpufreq.c +++ b/drivers/cpufreq/tegra186-cpufreq.c @@ -117,7 +117,7 @@ static unsigned int tegra186_cpufreq_get(unsigned int cpu) static struct cpufreq_driver tegra186_cpufreq_driver = { .name = "tegra186", - .flags = CPUFREQ_STICKY | CPUFREQ_HAVE_GOVERNOR_PER_POLICY | + .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, .get = tegra186_cpufreq_get, .verify = cpufreq_generic_frequency_table_verify, diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c index 6a67f36f3b80..a9620e4489ae 100644 --- a/drivers/cpufreq/tegra194-cpufreq.c +++ b/drivers/cpufreq/tegra194-cpufreq.c @@ -272,8 +272,7 @@ static int tegra194_cpufreq_set_target(struct cpufreq_policy *policy, static struct cpufreq_driver tegra194_cpufreq_driver = { .name = "tegra194", - .flags = CPUFREQ_STICKY | CPUFREQ_CONST_LOOPS | - CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_INITIAL_FREQ_CHECK, .verify = cpufreq_generic_frequency_table_verify, .target_index = tegra194_cpufreq_set_target, .get = tegra194_get_speed, diff --git a/drivers/cpufreq/vexpress-spc-cpufreq.c b/drivers/cpufreq/vexpress-spc-cpufreq.c index f711d8eaea6a..51dfa9ae6cf5 100644 --- a/drivers/cpufreq/vexpress-spc-cpufreq.c +++ b/drivers/cpufreq/vexpress-spc-cpufreq.c @@ -486,8 +486,7 @@ static void ve_spc_cpufreq_ready(struct cpufreq_policy *policy) static struct cpufreq_driver ve_spc_cpufreq_driver = { .name = "vexpress-spc", - .flags = CPUFREQ_STICKY | - CPUFREQ_HAVE_GOVERNOR_PER_POLICY | + .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, .verify = cpufreq_generic_frequency_table_verify, .target_index = ve_spc_cpufreq_set_target, diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 9c8b7437b6cd..c8e40e91fe9b 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -387,8 +387,13 @@ struct cpufreq_driver { /* flags */ -/* driver isn't removed even if all ->init() calls failed */ -#define CPUFREQ_STICKY BIT(0) +/* + * Set by drivers that need to update internale upper and lower boundaries along + * with the target frequency and so the core and governors should also invoke + * the diver if the target frequency does not change, but the policy min or max + * may have changed. + */ +#define CPUFREQ_NEED_UPDATE_LIMITS BIT(0) /* loops_per_jiffy or other kernel "constants" aren't affected by frequency transitions */ #define CPUFREQ_CONST_LOOPS BIT(1) @@ -432,14 +437,6 @@ struct cpufreq_driver { */ #define CPUFREQ_IS_COOLING_DEV BIT(7) -/* - * Set by drivers that need to update internale upper and lower boundaries along - * with the target frequency and so the core and governors should also invoke - * the diver if the target frequency does not change, but the policy min or max - * may have changed. - */ -#define CPUFREQ_NEED_UPDATE_LIMITS BIT(8) - int cpufreq_register_driver(struct cpufreq_driver *driver_data); int cpufreq_unregister_driver(struct cpufreq_driver *driver_data); -- cgit v1.2.3 From 2f0531869fd22182e769b10dd6cf151861ede791 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 2 Feb 2021 11:11:55 +0530 Subject: cpufreq: Remove unused flag CPUFREQ_PM_NO_WARN This flag is set by one of the drivers but it isn't used in the code otherwise. Remove the unused flag and update the driver. Signed-off-by: Viresh Kumar Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/pmac32-cpufreq.c | 3 +-- include/linux/cpufreq.h | 13 +++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/drivers/cpufreq/pmac32-cpufreq.c b/drivers/cpufreq/pmac32-cpufreq.c index 73621bc11976..4f20c6a9108d 100644 --- a/drivers/cpufreq/pmac32-cpufreq.c +++ b/drivers/cpufreq/pmac32-cpufreq.c @@ -439,8 +439,7 @@ static struct cpufreq_driver pmac_cpufreq_driver = { .init = pmac_cpufreq_cpu_init, .suspend = pmac_cpufreq_suspend, .resume = pmac_cpufreq_resume, - .flags = CPUFREQ_PM_NO_WARN | - CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING, + .flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING, .attr = cpufreq_generic_attr, .name = "powermac", }; diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index c8e40e91fe9b..353969c7acd3 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -398,8 +398,11 @@ struct cpufreq_driver { /* loops_per_jiffy or other kernel "constants" aren't affected by frequency transitions */ #define CPUFREQ_CONST_LOOPS BIT(1) -/* don't warn on suspend/resume speed mismatches */ -#define CPUFREQ_PM_NO_WARN BIT(2) +/* + * Set by drivers that want the core to automatically register the cpufreq + * driver as a thermal cooling device. + */ +#define CPUFREQ_IS_COOLING_DEV BIT(2) /* * This should be set by platforms having multiple clock-domains, i.e. @@ -431,12 +434,6 @@ struct cpufreq_driver { */ #define CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING BIT(6) -/* - * Set by drivers that want the core to automatically register the cpufreq - * driver as a thermal cooling device. - */ -#define CPUFREQ_IS_COOLING_DEV BIT(7) - int cpufreq_register_driver(struct cpufreq_driver *driver_data); int cpufreq_unregister_driver(struct cpufreq_driver *driver_data); -- cgit v1.2.3 From 7e3ce05e7f650371061d0b9eec1e1cf74ed6fca0 Mon Sep 17 00:00:00 2001 From: Marcelo Ricardo Leitner Date: Wed, 3 Feb 2021 22:48:16 -0300 Subject: netlink: add tracepoint at NL_SET_ERR_MSG Often userspace won't request the extack information, or they don't log it because of log level or so, and even when they do, sometimes it's not enough to know exactly what caused the error. Netlink extack is the standard way of reporting erros with descriptive error messages. With a trace point on it, we then can know exactly where the error happened, regardless of userspace app. Also, we can even see if the err msg was overwritten. The wrapper do_trace_netlink_extack() is because trace points shouldn't be called from .h files, as trace points are not that small, and the function call to do_trace_netlink_extack() on the macros is not protected by tracepoint_enabled() because the macros are called from modules, and this would require exporting some trace structs. As this is error path, it's better to export just the wrapper instead. v2: removed leftover tracepoint declaration Signed-off-by: Marcelo Ricardo Leitner Reviewed-by: David Ahern Link: https://lore.kernel.org/r/4546b63e67b2989789d146498b13cc09e1fdc543.1612403190.git.marcelo.leitner@gmail.com Signed-off-by: Jakub Kicinski --- include/linux/netlink.h | 6 ++++++ include/trace/events/netlink.h | 29 +++++++++++++++++++++++++++++ net/netlink/af_netlink.c | 8 ++++++++ 3 files changed, 43 insertions(+) create mode 100644 include/trace/events/netlink.h (limited to 'include/linux') diff --git a/include/linux/netlink.h b/include/linux/netlink.h index 9f118771e248..0bcf98098c5a 100644 --- a/include/linux/netlink.h +++ b/include/linux/netlink.h @@ -11,6 +11,8 @@ struct net; +void do_trace_netlink_extack(const char *msg); + static inline struct nlmsghdr *nlmsg_hdr(const struct sk_buff *skb) { return (struct nlmsghdr *)skb->data; @@ -90,6 +92,8 @@ struct netlink_ext_ack { static const char __msg[] = msg; \ struct netlink_ext_ack *__extack = (extack); \ \ + do_trace_netlink_extack(__msg); \ + \ if (__extack) \ __extack->_msg = __msg; \ } while (0) @@ -110,6 +114,8 @@ struct netlink_ext_ack { static const char __msg[] = msg; \ struct netlink_ext_ack *__extack = (extack); \ \ + do_trace_netlink_extack(__msg); \ + \ if (__extack) { \ __extack->_msg = __msg; \ __extack->bad_attr = (attr); \ diff --git a/include/trace/events/netlink.h b/include/trace/events/netlink.h new file mode 100644 index 000000000000..3b7be3b386a4 --- /dev/null +++ b/include/trace/events/netlink.h @@ -0,0 +1,29 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM netlink + +#if !defined(_TRACE_NETLINK_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_NETLINK_H + +#include + +TRACE_EVENT(netlink_extack, + + TP_PROTO(const char *msg), + + TP_ARGS(msg), + + TP_STRUCT__entry( + __string( msg, msg ) + ), + + TP_fast_assign( + __assign_str(msg, msg); + ), + + TP_printk("msg=%s", __get_str(msg)) +); + +#endif /* _TRACE_NETLINK_H */ + +/* This part must be outside protection */ +#include diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index daca50d6bb12..dd488938447f 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -67,6 +67,8 @@ #include #include #include +#define CREATE_TRACE_POINTS +#include #include "af_netlink.h" @@ -147,6 +149,12 @@ static BLOCKING_NOTIFIER_HEAD(netlink_chain); static const struct rhashtable_params netlink_rhashtable_params; +void do_trace_netlink_extack(const char *msg) +{ + trace_netlink_extack(msg); +} +EXPORT_SYMBOL(do_trace_netlink_extack); + static inline u32 netlink_group_mask(u32 group) { return group ? 1 << (group - 1) : 0; -- cgit v1.2.3 From 0053859496baa17c7675526936677c9213bf5a0d Mon Sep 17 00:00:00 2001 From: Brian Vazquez Date: Thu, 4 Feb 2021 18:18:38 +0000 Subject: net: add EXPORT_INDIRECT_CALLABLE wrapper When a static function is annotated with INDIRECT_CALLABLE_SCOPE and CONFIG_RETPOLINE is set, the static keyword is removed. Sometimes the function needs to be exported but EXPORT_SYMBOL can't be used because if CONFIG_RETPOLINE is not set, we will attempt to export a static symbol. This patch introduces a new indirect call wrapper: EXPORT_INDIRECT_CALLABLE. This basically does EXPORT_SYMBOL when CONFIG_RETPOLINE is set, but does nothing when it's not. Reported-by: Stephen Rothwell Signed-off-by: Brian Vazquez Link: https://lore.kernel.org/r/20210204181839.558951-1-brianvv@google.com Signed-off-by: Jakub Kicinski --- include/linux/indirect_call_wrapper.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/indirect_call_wrapper.h b/include/linux/indirect_call_wrapper.h index 54c02c84906a..a8345c8a613d 100644 --- a/include/linux/indirect_call_wrapper.h +++ b/include/linux/indirect_call_wrapper.h @@ -36,6 +36,7 @@ #define INDIRECT_CALLABLE_DECLARE(f) f #define INDIRECT_CALLABLE_SCOPE +#define EXPORT_INDIRECT_CALLABLE(f) EXPORT_SYMBOL(f) #else #define INDIRECT_CALL_1(f, f1, ...) f(__VA_ARGS__) @@ -44,6 +45,7 @@ #define INDIRECT_CALL_4(f, f4, f3, f2, f1, ...) f(__VA_ARGS__) #define INDIRECT_CALLABLE_DECLARE(f) #define INDIRECT_CALLABLE_SCOPE static +#define EXPORT_INDIRECT_CALLABLE(f) #endif /* -- cgit v1.2.3 From 1d7bab6a94458e959f3f55788fd50ddc7d97403b Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 2 Feb 2021 13:30:54 +0000 Subject: mm: constify page_is_pfmemalloc() argument The function only tests for page->index, so its argument should be const. Signed-off-by: Alexander Lobakin Reviewed-by: Jesse Brandeburg Acked-by: David Rientjes Signed-off-by: Jakub Kicinski --- include/linux/mm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index ecdf8a8cd6ae..078633d43af9 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1584,7 +1584,7 @@ struct address_space *page_mapping_file(struct page *page); * ALLOC_NO_WATERMARKS and the low watermark was not * met implying that the system is under some pressure. */ -static inline bool page_is_pfmemalloc(struct page *page) +static inline bool page_is_pfmemalloc(const struct page *page) { /* * Page index cannot be this large so this must be -- cgit v1.2.3 From 48f971c9c80a728646fc03367a28df747f20d0f4 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 2 Feb 2021 13:31:05 +0000 Subject: skbuff: constify skb_propagate_pfmemalloc() "page" argument The function doesn't write anything to the page struct itself, so this argument can be const. Misc: align second argument to the brace while at it. Signed-off-by: Alexander Lobakin Reviewed-by: Jesse Brandeburg Acked-by: David Rientjes Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 9313b5aaf45b..b027526da4f9 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2943,8 +2943,8 @@ static inline struct page *dev_alloc_page(void) * @page: The page that was allocated from skb_alloc_page * @skb: The skb that may need pfmemalloc set */ -static inline void skb_propagate_pfmemalloc(struct page *page, - struct sk_buff *skb) +static inline void skb_propagate_pfmemalloc(const struct page *page, + struct sk_buff *skb) { if (page_is_pfmemalloc(page)) skb->pfmemalloc = true; -- cgit v1.2.3 From bc38f30f8dbce0afb8af05d917bee084b1329418 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Tue, 2 Feb 2021 13:31:21 +0000 Subject: net: introduce common dev_page_is_reusable() A bunch of drivers test the page before reusing/recycling for two common conditions: - if a page was allocated under memory pressure (pfmemalloc page); - if a page was allocated at a distant memory node (to exclude slowdowns). Introduce a new common inline for doing this, with likely() already folded inside to make driver code a bit simpler. Suggested-by: David Rientjes Suggested-by: Jakub Kicinski Cc: John Hubbard Signed-off-by: Alexander Lobakin Reviewed-by: Jesse Brandeburg Acked-by: David Rientjes Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index b027526da4f9..0e42c53b8ca9 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2938,6 +2938,22 @@ static inline struct page *dev_alloc_page(void) return dev_alloc_pages(0); } +/** + * dev_page_is_reusable - check whether a page can be reused for network Rx + * @page: the page to test + * + * A page shouldn't be considered for reusing/recycling if it was allocated + * under memory pressure or at a distant memory node. + * + * Returns false if this page should be returned to page allocator, true + * otherwise. + */ +static inline bool dev_page_is_reusable(const struct page *page) +{ + return likely(page_to_nid(page) == numa_mem_id() && + !page_is_pfmemalloc(page)); +} + /** * skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page * @page: The page that was allocated from skb_alloc_page -- cgit v1.2.3 From 1faba27f11c8da244e793546a1b35a9b1da8208e Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Wed, 3 Feb 2021 15:51:09 +0200 Subject: ipv6: silence compilation warning for non-IPV6 builds The W=1 compilation of allmodconfig generates the following warning: net/ipv6/icmp.c:448:6: warning: no previous prototype for 'icmp6_send' [-Wmissing-prototypes] 448 | void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info, | ^~~~~~~~~~ Fix it by providing function declaration for builds with ipv6 as a module. Signed-off-by: Leon Romanovsky Signed-off-by: Jakub Kicinski --- include/linux/icmpv6.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/icmpv6.h b/include/linux/icmpv6.h index 1b3371ae8193..452d8978ffc7 100644 --- a/include/linux/icmpv6.h +++ b/include/linux/icmpv6.h @@ -16,9 +16,9 @@ static inline struct icmp6hdr *icmp6_hdr(const struct sk_buff *skb) typedef void ip6_icmp_send_t(struct sk_buff *skb, u8 type, u8 code, __u32 info, const struct in6_addr *force_saddr); -#if IS_BUILTIN(CONFIG_IPV6) void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info, const struct in6_addr *force_saddr); +#if IS_BUILTIN(CONFIG_IPV6) static inline void icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info) { icmp6_send(skb, type, code, info, NULL); -- cgit v1.2.3 From 52cbd23a119c6ebf40a527e53f3402d2ea38eccb Mon Sep 17 00:00:00 2001 From: Willem de Bruijn Date: Wed, 3 Feb 2021 14:29:52 -0500 Subject: udp: fix skb_copy_and_csum_datagram with odd segment sizes When iteratively computing a checksum with csum_block_add, track the offset "pos" to correctly rotate in csum_block_add when offset is odd. The open coded implementation of skb_copy_and_csum_datagram did this. With the switch to __skb_datagram_iter calling csum_and_copy_to_iter, pos was reinitialized to 0 on each call. Bring back the pos by passing it along with the csum to the callback. Changes v1->v2 - pass csum value, instead of csump pointer (Alexander Duyck) Link: https://lore.kernel.org/netdev/20210128152353.GB27281@optiplex/ Fixes: 950fcaecd5cc ("datagram: consolidate datagram copy to iter helpers") Reported-by: Oliver Graute Signed-off-by: Willem de Bruijn Reviewed-by: Alexander Duyck Reviewed-by: Eric Dumazet Link: https://lore.kernel.org/r/20210203192952.1849843-1-willemdebruijn.kernel@gmail.com Signed-off-by: Jakub Kicinski --- include/linux/uio.h | 8 +++++++- lib/iov_iter.c | 24 ++++++++++++++---------- net/core/datagram.c | 12 ++++++++++-- 3 files changed, 31 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/uio.h b/include/linux/uio.h index 72d88566694e..27ff8eb786dc 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -260,7 +260,13 @@ static inline void iov_iter_reexpand(struct iov_iter *i, size_t count) { i->count = count; } -size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump, struct iov_iter *i); + +struct csum_state { + __wsum csum; + size_t off; +}; + +size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csstate, struct iov_iter *i); size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i); bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i); size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp, diff --git a/lib/iov_iter.c b/lib/iov_iter.c index a21e6a5792c5..f0b2ccb1bb01 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -592,14 +592,15 @@ static __wsum csum_and_memcpy(void *to, const void *from, size_t len, } static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes, - __wsum *csum, struct iov_iter *i) + struct csum_state *csstate, + struct iov_iter *i) { struct pipe_inode_info *pipe = i->pipe; unsigned int p_mask = pipe->ring_size - 1; + __wsum sum = csstate->csum; + size_t off = csstate->off; unsigned int i_head; size_t n, r; - size_t off = 0; - __wsum sum = *csum; if (!sanity(i)) return 0; @@ -621,7 +622,8 @@ static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes, i_head++; } while (n); i->count -= bytes; - *csum = sum; + csstate->csum = sum; + csstate->off = off; return bytes; } @@ -1522,18 +1524,19 @@ bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, } EXPORT_SYMBOL(csum_and_copy_from_iter_full); -size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump, +size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, struct iov_iter *i) { + struct csum_state *csstate = _csstate; const char *from = addr; - __wsum *csum = csump; __wsum sum, next; - size_t off = 0; + size_t off; if (unlikely(iov_iter_is_pipe(i))) - return csum_and_copy_to_pipe_iter(addr, bytes, csum, i); + return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i); - sum = *csum; + sum = csstate->csum; + off = csstate->off; if (unlikely(iov_iter_is_discard(i))) { WARN_ON(1); /* for now */ return 0; @@ -1561,7 +1564,8 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump, off += v.iov_len; }) ) - *csum = sum; + csstate->csum = sum; + csstate->off = off; return bytes; } EXPORT_SYMBOL(csum_and_copy_to_iter); diff --git a/net/core/datagram.c b/net/core/datagram.c index 81809fa735a7..15ab9ffb27fe 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -721,8 +721,16 @@ static int skb_copy_and_csum_datagram(const struct sk_buff *skb, int offset, struct iov_iter *to, int len, __wsum *csump) { - return __skb_datagram_iter(skb, offset, to, len, true, - csum_and_copy_to_iter, csump); + struct csum_state csdata = { .csum = *csump }; + int ret; + + ret = __skb_datagram_iter(skb, offset, to, len, true, + csum_and_copy_to_iter, &csdata); + if (ret) + return ret; + + *csump = csdata.csum; + return 0; } /** -- cgit v1.2.3 From 151f6ff78cdf1d6de76e90556cfc43f1e48abe18 Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Thu, 4 Feb 2021 17:17:06 +0300 Subject: software node: Provide replacement for device_add_properties() At the moment the function device_del() is calling device_remove_properties() unconditionally. That will result into the reference count of the software node attached to the device being decremented, and in most cases it will hit 0 at that point. So in practice device_del() will unregister the software node attached to the device, even if that was not the intention of the caller. Right now software nodes can not be reused or shared because of that. So device_del() can not unregister the software nodes unconditionally like that. Unfortunately some of the users of device_add_properties() are now relying on this behaviour. Because of that, and also in general, we do need a function that can offer similar behaviour where the lifetime of the software node is bound to the lifetime of the device. But it just has to be a separate function so the behaviour is optional. We can not remove the device_remove_properties() call from device_del() before we have that new function, and before we have replaced device_add_properties() calls with it in all the places that require that behaviour. This adds function device_create_managed_software_node() that can be used for exactly that purpose. Software nodes created with it are declared "managed", and separate handling for those nodes is added to the software node code. The reference count of the "managed" nodes is decremented when the device they are attached to is removed. This will not affect the other nodes that are not declared "managed". The function device_create_managed_software_node() has also one additional feature that device_add_properties() does not have. It allows the software nodes created with it to be part of a node hierarchy by taking also an optional parent node as parameter. Reviewed-by: Rafael J. Wysocki Signed-off-by: Heikki Krogerus Link: https://lore.kernel.org/r/20210204141711.53775-2-heikki.krogerus@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/base/swnode.c | 43 +++++++++++++++++++++++++++++++++++++++++++ include/linux/property.h | 4 ++++ 2 files changed, 47 insertions(+) (limited to 'include/linux') diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c index 4842dd678200..45cf8c879d71 100644 --- a/drivers/base/swnode.c +++ b/drivers/base/swnode.c @@ -24,6 +24,7 @@ struct swnode { struct swnode *parent; unsigned int allocated:1; + unsigned int managed:1; }; static DEFINE_IDA(swnode_root_ids); @@ -903,6 +904,43 @@ void device_remove_software_node(struct device *dev) } EXPORT_SYMBOL_GPL(device_remove_software_node); +/** + * device_create_managed_software_node - Create a software node for a device + * @dev: The device the software node is assigned to. + * @properties: Device properties for the software node. + * @parent: Parent of the software node. + * + * Creates a software node as a managed resource for @dev, which means the + * lifetime of the newly created software node is tied to the lifetime of @dev. + * Software nodes created with this function should not be reused or shared + * because of that. The function takes a deep copy of @properties for the + * software node. + * + * Since the new software node is assigned directly to @dev, and since it should + * not be shared, it is not returned to the caller. The function returns 0 on + * success, and errno in case of an error. + */ +int device_create_managed_software_node(struct device *dev, + const struct property_entry *properties, + const struct software_node *parent) +{ + struct fwnode_handle *p = software_node_fwnode(parent); + struct fwnode_handle *fwnode; + + if (parent && !p) + return -EINVAL; + + fwnode = fwnode_create_software_node(properties, p); + if (IS_ERR(fwnode)) + return PTR_ERR(fwnode); + + to_swnode(fwnode)->managed = true; + set_secondary_fwnode(dev, fwnode); + + return 0; +} +EXPORT_SYMBOL_GPL(device_create_managed_software_node); + int software_node_notify(struct device *dev, unsigned long action) { struct swnode *swnode; @@ -931,6 +969,11 @@ int software_node_notify(struct device *dev, unsigned long action) sysfs_remove_link(&swnode->kobj, dev_name(dev)); sysfs_remove_link(&dev->kobj, "software_node"); kobject_put(&swnode->kobj); + + if (swnode->managed) { + set_secondary_fwnode(dev, NULL); + kobject_put(&swnode->kobj); + } break; default: break; diff --git a/include/linux/property.h b/include/linux/property.h index b0e413dc5927..dafccfce0262 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -491,4 +491,8 @@ void fwnode_remove_software_node(struct fwnode_handle *fwnode); int device_add_software_node(struct device *dev, const struct software_node *swnode); void device_remove_software_node(struct device *dev); +int device_create_managed_software_node(struct device *dev, + const struct property_entry *properties, + const struct software_node *parent); + #endif /* _LINUX_PROPERTY_H_ */ -- cgit v1.2.3 From a8c3209998afb5c4941b49e35b513cea9050cb4a Mon Sep 17 00:00:00 2001 From: Andres Beltran Date: Tue, 8 Dec 2020 05:53:11 +0100 Subject: Drivers: hv: vmbus: Copy packets sent by Hyper-V out of the ring buffer Pointers to ring-buffer packets sent by Hyper-V are used within the guest VM. Hyper-V can send packets with erroneous values or modify packet fields after they are processed by the guest. To defend against these scenarios, return a copy of the incoming VMBus packet after validating its length and offset fields in hv_pkt_iter_first(). In this way, the packet can no longer be modified by the host. Signed-off-by: Andres Beltran Co-developed-by: Andrea Parri (Microsoft) Signed-off-by: Andrea Parri (Microsoft) Cc: "David S. Miller" Cc: Jakub Kicinski Cc: "James E.J. Bottomley" Cc: "Martin K. Petersen" Cc: netdev@vger.kernel.org Cc: linux-scsi@vger.kernel.org Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20201208045311.10244-1-parri.andrea@gmail.com Signed-off-by: Wei Liu --- drivers/hv/channel.c | 9 +++-- drivers/hv/hv_fcopy.c | 1 + drivers/hv/hv_kvp.c | 1 + drivers/hv/hyperv_vmbus.h | 2 +- drivers/hv/ring_buffer.c | 82 +++++++++++++++++++++++++++++++++------ drivers/net/hyperv/hyperv_net.h | 3 ++ drivers/net/hyperv/netvsc.c | 2 + drivers/net/hyperv/rndis_filter.c | 2 + drivers/scsi/storvsc_drv.c | 10 +++++ include/linux/hyperv.h | 48 +++++++++++++++++++---- net/vmw_vsock/hyperv_transport.c | 4 +- 11 files changed, 139 insertions(+), 25 deletions(-) (limited to 'include/linux') diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index 6fb0c76bfbf8..0d63862d6551 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -597,12 +597,15 @@ static int __vmbus_open(struct vmbus_channel *newchannel, newchannel->onchannel_callback = onchannelcallback; newchannel->channel_callback_context = context; - err = hv_ringbuffer_init(&newchannel->outbound, page, send_pages); + if (!newchannel->max_pkt_size) + newchannel->max_pkt_size = VMBUS_DEFAULT_MAX_PKT_SIZE; + + err = hv_ringbuffer_init(&newchannel->outbound, page, send_pages, 0); if (err) goto error_clean_ring; - err = hv_ringbuffer_init(&newchannel->inbound, - &page[send_pages], recv_pages); + err = hv_ringbuffer_init(&newchannel->inbound, &page[send_pages], + recv_pages, newchannel->max_pkt_size); if (err) goto error_clean_ring; diff --git a/drivers/hv/hv_fcopy.c b/drivers/hv/hv_fcopy.c index 5040d7e0cd9e..85784a9e6a36 100644 --- a/drivers/hv/hv_fcopy.c +++ b/drivers/hv/hv_fcopy.c @@ -329,6 +329,7 @@ int hv_fcopy_init(struct hv_util_service *srv) { recv_buffer = srv->recv_buffer; fcopy_transaction.recv_channel = srv->channel; + fcopy_transaction.recv_channel->max_pkt_size = HV_HYP_PAGE_SIZE * 2; /* * When this driver loads, the user level daemon that diff --git a/drivers/hv/hv_kvp.c b/drivers/hv/hv_kvp.c index 754d35a25a1c..7d2a7b918847 100644 --- a/drivers/hv/hv_kvp.c +++ b/drivers/hv/hv_kvp.c @@ -741,6 +741,7 @@ hv_kvp_init(struct hv_util_service *srv) { recv_buffer = srv->recv_buffer; kvp_transaction.recv_channel = srv->channel; + kvp_transaction.recv_channel->max_pkt_size = HV_HYP_PAGE_SIZE * 4; /* * When this driver loads, the user level daemon that diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h index 9416e09ebd58..42f3d9d123a1 100644 --- a/drivers/hv/hyperv_vmbus.h +++ b/drivers/hv/hyperv_vmbus.h @@ -174,7 +174,7 @@ extern int hv_synic_cleanup(unsigned int cpu); void hv_ringbuffer_pre_init(struct vmbus_channel *channel); int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info, - struct page *pages, u32 pagecnt); + struct page *pages, u32 pagecnt, u32 max_pkt_size); void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info); diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c index 35833d4d1a1d..29e90477363a 100644 --- a/drivers/hv/ring_buffer.c +++ b/drivers/hv/ring_buffer.c @@ -190,7 +190,7 @@ void hv_ringbuffer_pre_init(struct vmbus_channel *channel) /* Initialize the ring buffer. */ int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info, - struct page *pages, u32 page_cnt) + struct page *pages, u32 page_cnt, u32 max_pkt_size) { int i; struct page **pages_wraparound; @@ -232,6 +232,14 @@ int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info, sizeof(struct hv_ring_buffer); ring_info->priv_read_index = 0; + /* Initialize buffer that holds copies of incoming packets */ + if (max_pkt_size) { + ring_info->pkt_buffer = kzalloc(max_pkt_size, GFP_KERNEL); + if (!ring_info->pkt_buffer) + return -ENOMEM; + ring_info->pkt_buffer_size = max_pkt_size; + } + spin_lock_init(&ring_info->ring_lock); return 0; @@ -244,6 +252,9 @@ void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info) vunmap(ring_info->ring_buffer); ring_info->ring_buffer = NULL; mutex_unlock(&ring_info->ring_buffer_mutex); + + kfree(ring_info->pkt_buffer); + ring_info->pkt_buffer_size = 0; } /* Write to the ring buffer. */ @@ -385,7 +396,7 @@ int hv_ringbuffer_read(struct vmbus_channel *channel, memcpy(buffer, (const char *)desc + offset, packetlen); /* Advance ring index to next packet descriptor */ - __hv_pkt_iter_next(channel, desc); + __hv_pkt_iter_next(channel, desc, true); /* Notify host of update */ hv_pkt_iter_close(channel); @@ -411,6 +422,22 @@ static u32 hv_pkt_iter_avail(const struct hv_ring_buffer_info *rbi) return (rbi->ring_datasize - priv_read_loc) + write_loc; } +/* + * Get first vmbus packet without copying it out of the ring buffer + */ +struct vmpacket_descriptor *hv_pkt_iter_first_raw(struct vmbus_channel *channel) +{ + struct hv_ring_buffer_info *rbi = &channel->inbound; + + hv_debug_delay_test(channel, MESSAGE_DELAY); + + if (hv_pkt_iter_avail(rbi) < sizeof(struct vmpacket_descriptor)) + return NULL; + + return (struct vmpacket_descriptor *)(hv_get_ring_buffer(rbi) + rbi->priv_read_index); +} +EXPORT_SYMBOL_GPL(hv_pkt_iter_first_raw); + /* * Get first vmbus packet from ring buffer after read_index * @@ -419,17 +446,49 @@ static u32 hv_pkt_iter_avail(const struct hv_ring_buffer_info *rbi) struct vmpacket_descriptor *hv_pkt_iter_first(struct vmbus_channel *channel) { struct hv_ring_buffer_info *rbi = &channel->inbound; - struct vmpacket_descriptor *desc; + struct vmpacket_descriptor *desc, *desc_copy; + u32 bytes_avail, pkt_len, pkt_offset; - hv_debug_delay_test(channel, MESSAGE_DELAY); - if (hv_pkt_iter_avail(rbi) < sizeof(struct vmpacket_descriptor)) + desc = hv_pkt_iter_first_raw(channel); + if (!desc) return NULL; - desc = hv_get_ring_buffer(rbi) + rbi->priv_read_index; - if (desc) - prefetch((char *)desc + (desc->len8 << 3)); + bytes_avail = min(rbi->pkt_buffer_size, hv_pkt_iter_avail(rbi)); + + /* + * Ensure the compiler does not use references to incoming Hyper-V values (which + * could change at any moment) when reading local variables later in the code + */ + pkt_len = READ_ONCE(desc->len8) << 3; + pkt_offset = READ_ONCE(desc->offset8) << 3; + + /* + * If pkt_len is invalid, set it to the smaller of hv_pkt_iter_avail() and + * rbi->pkt_buffer_size + */ + if (pkt_len < sizeof(struct vmpacket_descriptor) || pkt_len > bytes_avail) + pkt_len = bytes_avail; + + /* + * If pkt_offset is invalid, arbitrarily set it to + * the size of vmpacket_descriptor + */ + if (pkt_offset < sizeof(struct vmpacket_descriptor) || pkt_offset > pkt_len) + pkt_offset = sizeof(struct vmpacket_descriptor); + + /* Copy the Hyper-V packet out of the ring buffer */ + desc_copy = (struct vmpacket_descriptor *)rbi->pkt_buffer; + memcpy(desc_copy, desc, pkt_len); + + /* + * Hyper-V could still change len8 and offset8 after the earlier read. + * Ensure that desc_copy has legal values for len8 and offset8 that + * are consistent with the copy we just made + */ + desc_copy->len8 = pkt_len >> 3; + desc_copy->offset8 = pkt_offset >> 3; - return desc; + return desc_copy; } EXPORT_SYMBOL_GPL(hv_pkt_iter_first); @@ -441,7 +500,8 @@ EXPORT_SYMBOL_GPL(hv_pkt_iter_first); */ struct vmpacket_descriptor * __hv_pkt_iter_next(struct vmbus_channel *channel, - const struct vmpacket_descriptor *desc) + const struct vmpacket_descriptor *desc, + bool copy) { struct hv_ring_buffer_info *rbi = &channel->inbound; u32 packetlen = desc->len8 << 3; @@ -454,7 +514,7 @@ __hv_pkt_iter_next(struct vmbus_channel *channel, rbi->priv_read_index -= dsize; /* more data? */ - return hv_pkt_iter_first(channel); + return copy ? hv_pkt_iter_first(channel) : hv_pkt_iter_first_raw(channel); } EXPORT_SYMBOL_GPL(__hv_pkt_iter_next); diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h index 2a87cfa27ac0..7ea6936f86ef 100644 --- a/drivers/net/hyperv/hyperv_net.h +++ b/drivers/net/hyperv/hyperv_net.h @@ -860,9 +860,12 @@ static inline u32 netvsc_rqstor_size(unsigned long ringbytes) ringbytes / NETVSC_MIN_IN_MSG_SIZE; } +#define NETVSC_MAX_XFER_PAGE_RANGES 375 #define NETVSC_XFER_HEADER_SIZE(rng_cnt) \ (offsetof(struct vmtransfer_page_packet_header, ranges) + \ (rng_cnt) * sizeof(struct vmtransfer_page_range)) +#define NETVSC_MAX_PKT_SIZE (NETVSC_XFER_HEADER_SIZE(NETVSC_MAX_XFER_PAGE_RANGES) + \ + sizeof(struct nvsp_message) + (sizeof(u32) * VRSS_SEND_TAB_SIZE)) struct multi_send_data { struct sk_buff *skb; /* skb containing the pkt */ diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c index 2350342b961f..1510a236aa34 100644 --- a/drivers/net/hyperv/netvsc.c +++ b/drivers/net/hyperv/netvsc.c @@ -1530,6 +1530,8 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device, /* Open the channel */ device->channel->rqstor_size = netvsc_rqstor_size(netvsc_ring_bytes); + device->channel->max_pkt_size = NETVSC_MAX_PKT_SIZE; + ret = vmbus_open(device->channel, netvsc_ring_bytes, netvsc_ring_bytes, NULL, 0, netvsc_channel_cb, net_device->chan_table); diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c index 598713c0d5a8..7e6dee2f02a4 100644 --- a/drivers/net/hyperv/rndis_filter.c +++ b/drivers/net/hyperv/rndis_filter.c @@ -1174,6 +1174,8 @@ static void netvsc_sc_open(struct vmbus_channel *new_sc) nvchan->channel = new_sc; new_sc->rqstor_size = netvsc_rqstor_size(netvsc_ring_bytes); + new_sc->max_pkt_size = NETVSC_MAX_PKT_SIZE; + ret = vmbus_open(new_sc, netvsc_ring_bytes, netvsc_ring_bytes, NULL, 0, netvsc_channel_cb, nvchan); diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c index 2e4fa77445fd..7e59284dbf5b 100644 --- a/drivers/scsi/storvsc_drv.c +++ b/drivers/scsi/storvsc_drv.c @@ -414,6 +414,14 @@ static void storvsc_on_channel_callback(void *context); #define STORVSC_IDE_MAX_TARGETS 1 #define STORVSC_IDE_MAX_CHANNELS 1 +/* + * Upper bound on the size of a storvsc packet. vmscsi_size_delta is not + * included in the calculation because it is set after STORVSC_MAX_PKT_SIZE + * is used in storvsc_connect_to_vsp + */ +#define STORVSC_MAX_PKT_SIZE (sizeof(struct vmpacket_descriptor) +\ + sizeof(struct vstor_packet)) + struct storvsc_cmd_request { struct scsi_cmnd *cmd; @@ -698,6 +706,7 @@ static void handle_sc_creation(struct vmbus_channel *new_sc) return; memset(&props, 0, sizeof(struct vmstorage_channel_properties)); + new_sc->max_pkt_size = STORVSC_MAX_PKT_SIZE; /* * The size of vmbus_requestor is an upper bound on the number of requests @@ -1280,6 +1289,7 @@ static int storvsc_connect_to_vsp(struct hv_device *device, u32 ring_size, memset(&props, 0, sizeof(struct vmstorage_channel_properties)); + device->channel->max_pkt_size = STORVSC_MAX_PKT_SIZE; /* * The size of vmbus_requestor is an upper bound on the number of requests * that can be in-progress at any one time across all channels. diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 5ddb479c4d4c..fbae8406d5d4 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -181,6 +181,10 @@ struct hv_ring_buffer_info { * being freed while the ring buffer is being accessed. */ struct mutex ring_buffer_mutex; + + /* Buffer that holds a copy of an incoming host packet */ + void *pkt_buffer; + u32 pkt_buffer_size; }; @@ -787,6 +791,8 @@ struct vmbus_device { bool perf_device; }; +#define VMBUS_DEFAULT_MAX_PKT_SIZE 4096 + struct vmbus_channel { struct list_head listentry; @@ -1008,6 +1014,9 @@ struct vmbus_channel { /* request/transaction ids for VMBus */ struct vmbus_requestor requestor; u32 rqstor_size; + + /* The max size of a packet on this channel */ + u32 max_pkt_size; }; u64 vmbus_next_request_id(struct vmbus_requestor *rqstor, u64 rqst_addr); @@ -1642,32 +1651,55 @@ static inline u32 hv_pkt_datalen(const struct vmpacket_descriptor *desc) } +struct vmpacket_descriptor * +hv_pkt_iter_first_raw(struct vmbus_channel *channel); + struct vmpacket_descriptor * hv_pkt_iter_first(struct vmbus_channel *channel); struct vmpacket_descriptor * __hv_pkt_iter_next(struct vmbus_channel *channel, - const struct vmpacket_descriptor *pkt); + const struct vmpacket_descriptor *pkt, + bool copy); void hv_pkt_iter_close(struct vmbus_channel *channel); -/* - * Get next packet descriptor from iterator - * If at end of list, return NULL and update host. - */ static inline struct vmpacket_descriptor * -hv_pkt_iter_next(struct vmbus_channel *channel, - const struct vmpacket_descriptor *pkt) +hv_pkt_iter_next_pkt(struct vmbus_channel *channel, + const struct vmpacket_descriptor *pkt, + bool copy) { struct vmpacket_descriptor *nxt; - nxt = __hv_pkt_iter_next(channel, pkt); + nxt = __hv_pkt_iter_next(channel, pkt, copy); if (!nxt) hv_pkt_iter_close(channel); return nxt; } +/* + * Get next packet descriptor without copying it out of the ring buffer + * If at end of list, return NULL and update host. + */ +static inline struct vmpacket_descriptor * +hv_pkt_iter_next_raw(struct vmbus_channel *channel, + const struct vmpacket_descriptor *pkt) +{ + return hv_pkt_iter_next_pkt(channel, pkt, false); +} + +/* + * Get next packet descriptor from iterator + * If at end of list, return NULL and update host. + */ +static inline struct vmpacket_descriptor * +hv_pkt_iter_next(struct vmbus_channel *channel, + const struct vmpacket_descriptor *pkt) +{ + return hv_pkt_iter_next_pkt(channel, pkt, true); +} + #define foreach_vmbus_pkt(pkt, channel) \ for (pkt = hv_pkt_iter_first(channel); pkt; \ pkt = hv_pkt_iter_next(channel, pkt)) diff --git a/net/vmw_vsock/hyperv_transport.c b/net/vmw_vsock/hyperv_transport.c index 630b851f8150..cd8b7c1ca9f1 100644 --- a/net/vmw_vsock/hyperv_transport.c +++ b/net/vmw_vsock/hyperv_transport.c @@ -600,7 +600,7 @@ static ssize_t hvs_stream_dequeue(struct vsock_sock *vsk, struct msghdr *msg, return -EOPNOTSUPP; if (need_refill) { - hvs->recv_desc = hv_pkt_iter_first(hvs->chan); + hvs->recv_desc = hv_pkt_iter_first_raw(hvs->chan); ret = hvs_update_recv_data(hvs); if (ret) return ret; @@ -614,7 +614,7 @@ static ssize_t hvs_stream_dequeue(struct vsock_sock *vsk, struct msghdr *msg, hvs->recv_data_len -= to_read; if (hvs->recv_data_len == 0) { - hvs->recv_desc = hv_pkt_iter_next(hvs->chan, hvs->recv_desc); + hvs->recv_desc = hv_pkt_iter_next_raw(hvs->chan, hvs->recv_desc); if (hvs->recv_desc) { ret = hvs_update_recv_data(hvs); if (ret) -- cgit v1.2.3 From 06caa778d8b2fbcb4ac3878751e39d116424ba9b Mon Sep 17 00:00:00 2001 From: Andres Beltran Date: Mon, 9 Nov 2020 11:07:04 +0100 Subject: hv_utils: Add validation for untrusted Hyper-V values For additional robustness in the face of Hyper-V errors or malicious behavior, validate all values that originate from packets that Hyper-V has sent to the guest in the host-to-guest ring buffer. Ensure that invalid values cannot cause indexing off the end of the icversion_data array in vmbus_prep_negotiate_resp(). Signed-off-by: Andres Beltran Co-developed-by: Andrea Parri (Microsoft) Signed-off-by: Andrea Parri (Microsoft) Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20201109100704.9152-1-parri.andrea@gmail.com Signed-off-by: Wei Liu --- drivers/hv/channel_mgmt.c | 24 +++-- drivers/hv/hv_fcopy.c | 36 ++++++-- drivers/hv/hv_kvp.c | 122 ++++++++++++++----------- drivers/hv/hv_snapshot.c | 89 +++++++++++-------- drivers/hv/hv_util.c | 222 ++++++++++++++++++++++++++++------------------ include/linux/hyperv.h | 9 +- 6 files changed, 314 insertions(+), 188 deletions(-) (limited to 'include/linux') diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 1d44bb635bb8..5bc5eef5da15 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -190,6 +190,7 @@ static u16 hv_get_dev_type(const struct vmbus_channel *channel) * vmbus_prep_negotiate_resp() - Create default response for Negotiate message * @icmsghdrp: Pointer to msg header structure * @buf: Raw buffer channel data + * @buflen: Length of the raw buffer channel data. * @fw_version: The framework versions we can support. * @fw_vercnt: The size of @fw_version. * @srv_version: The service versions we can support. @@ -202,8 +203,8 @@ static u16 hv_get_dev_type(const struct vmbus_channel *channel) * Set up and fill in default negotiate response message. * Mainly used by Hyper-V drivers. */ -bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, - u8 *buf, const int *fw_version, int fw_vercnt, +bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf, + u32 buflen, const int *fw_version, int fw_vercnt, const int *srv_version, int srv_vercnt, int *nego_fw_version, int *nego_srv_version) { @@ -215,10 +216,14 @@ bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, bool found_match = false; struct icmsg_negotiate *negop; + /* Check that there's enough space for icframe_vercnt, icmsg_vercnt */ + if (buflen < ICMSG_HDR + offsetof(struct icmsg_negotiate, reserved)) { + pr_err_ratelimited("Invalid icmsg negotiate\n"); + return false; + } + icmsghdrp->icmsgsize = 0x10; - negop = (struct icmsg_negotiate *)&buf[ - sizeof(struct vmbuspipe_hdr) + - sizeof(struct icmsg_hdr)]; + negop = (struct icmsg_negotiate *)&buf[ICMSG_HDR]; icframe_major = negop->icframe_vercnt; icframe_minor = 0; @@ -226,6 +231,15 @@ bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, icmsg_major = negop->icmsg_vercnt; icmsg_minor = 0; + /* Validate negop packet */ + if (icframe_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT || + icmsg_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT || + ICMSG_NEGOTIATE_PKT_SIZE(icframe_major, icmsg_major) > buflen) { + pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n", + icframe_major, icmsg_major); + goto fw_error; + } + /* * Select the framework version number we will * support. diff --git a/drivers/hv/hv_fcopy.c b/drivers/hv/hv_fcopy.c index 85784a9e6a36..660036da7449 100644 --- a/drivers/hv/hv_fcopy.c +++ b/drivers/hv/hv_fcopy.c @@ -235,15 +235,27 @@ void hv_fcopy_onchannelcallback(void *context) if (fcopy_transaction.state > HVUTIL_READY) return; - vmbus_recvpacket(channel, recv_buffer, HV_HYP_PAGE_SIZE * 2, &recvlen, - &requestid); - if (recvlen <= 0) + if (vmbus_recvpacket(channel, recv_buffer, HV_HYP_PAGE_SIZE * 2, &recvlen, &requestid)) { + pr_err_ratelimited("Fcopy request received. Could not read into recv buf\n"); return; + } + + if (!recvlen) + return; + + /* Ensure recvlen is big enough to read header data */ + if (recvlen < ICMSG_HDR) { + pr_err_ratelimited("Fcopy request received. Packet length too small: %d\n", + recvlen); + return; + } icmsghdr = (struct icmsg_hdr *)&recv_buffer[ sizeof(struct vmbuspipe_hdr)]; + if (icmsghdr->icmsgtype == ICMSGTYPE_NEGOTIATE) { - if (vmbus_prep_negotiate_resp(icmsghdr, recv_buffer, + if (vmbus_prep_negotiate_resp(icmsghdr, + recv_buffer, recvlen, fw_versions, FW_VER_COUNT, fcopy_versions, FCOPY_VER_COUNT, NULL, &fcopy_srv_version)) { @@ -252,10 +264,14 @@ void hv_fcopy_onchannelcallback(void *context) fcopy_srv_version >> 16, fcopy_srv_version & 0xFFFF); } - } else { - fcopy_msg = (struct hv_fcopy_hdr *)&recv_buffer[ - sizeof(struct vmbuspipe_hdr) + - sizeof(struct icmsg_hdr)]; + } else if (icmsghdr->icmsgtype == ICMSGTYPE_FCOPY) { + /* Ensure recvlen is big enough to contain hv_fcopy_hdr */ + if (recvlen < ICMSG_HDR + sizeof(struct hv_fcopy_hdr)) { + pr_err_ratelimited("Invalid Fcopy hdr. Packet length too small: %u\n", + recvlen); + return; + } + fcopy_msg = (struct hv_fcopy_hdr *)&recv_buffer[ICMSG_HDR]; /* * Stash away this global state for completing the @@ -280,6 +296,10 @@ void hv_fcopy_onchannelcallback(void *context) schedule_delayed_work(&fcopy_timeout_work, HV_UTIL_TIMEOUT * HZ); return; + } else { + pr_err_ratelimited("Fcopy request received. Invalid msg type: %d\n", + icmsghdr->icmsgtype); + return; } icmsghdr->icflags = ICMSGHDRFLAG_TRANSACTION | ICMSGHDRFLAG_RESPONSE; vmbus_sendpacket(channel, recv_buffer, recvlen, requestid, diff --git a/drivers/hv/hv_kvp.c b/drivers/hv/hv_kvp.c index 7d2a7b918847..c698592b83e4 100644 --- a/drivers/hv/hv_kvp.c +++ b/drivers/hv/hv_kvp.c @@ -662,71 +662,87 @@ void hv_kvp_onchannelcallback(void *context) if (kvp_transaction.state > HVUTIL_READY) return; - vmbus_recvpacket(channel, recv_buffer, HV_HYP_PAGE_SIZE * 4, &recvlen, - &requestid); - - if (recvlen > 0) { - icmsghdrp = (struct icmsg_hdr *)&recv_buffer[ - sizeof(struct vmbuspipe_hdr)]; - - if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { - if (vmbus_prep_negotiate_resp(icmsghdrp, - recv_buffer, fw_versions, FW_VER_COUNT, - kvp_versions, KVP_VER_COUNT, - NULL, &kvp_srv_version)) { - pr_info("KVP IC version %d.%d\n", - kvp_srv_version >> 16, - kvp_srv_version & 0xFFFF); - } - } else { - kvp_msg = (struct hv_kvp_msg *)&recv_buffer[ - sizeof(struct vmbuspipe_hdr) + - sizeof(struct icmsg_hdr)]; + if (vmbus_recvpacket(channel, recv_buffer, HV_HYP_PAGE_SIZE * 4, &recvlen, &requestid)) { + pr_err_ratelimited("KVP request received. Could not read into recv buf\n"); + return; + } - /* - * Stash away this global state for completing the - * transaction; note transactions are serialized. - */ + if (!recvlen) + return; - kvp_transaction.recv_len = recvlen; - kvp_transaction.recv_req_id = requestid; - kvp_transaction.kvp_msg = kvp_msg; + /* Ensure recvlen is big enough to read header data */ + if (recvlen < ICMSG_HDR) { + pr_err_ratelimited("KVP request received. Packet length too small: %d\n", + recvlen); + return; + } - if (kvp_transaction.state < HVUTIL_READY) { - /* Userspace is not registered yet */ - kvp_respond_to_host(NULL, HV_E_FAIL); - return; - } - kvp_transaction.state = HVUTIL_HOSTMSG_RECEIVED; + icmsghdrp = (struct icmsg_hdr *)&recv_buffer[sizeof(struct vmbuspipe_hdr)]; + + if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { + if (vmbus_prep_negotiate_resp(icmsghdrp, + recv_buffer, recvlen, + fw_versions, FW_VER_COUNT, + kvp_versions, KVP_VER_COUNT, + NULL, &kvp_srv_version)) { + pr_info("KVP IC version %d.%d\n", + kvp_srv_version >> 16, + kvp_srv_version & 0xFFFF); + } + } else if (icmsghdrp->icmsgtype == ICMSGTYPE_KVPEXCHANGE) { + /* + * recvlen is not checked against sizeof(struct kvp_msg) because kvp_msg contains + * a union of structs and the msg type received is not known. Code using this + * struct should provide validation when accessing its fields. + */ + kvp_msg = (struct hv_kvp_msg *)&recv_buffer[ICMSG_HDR]; - /* - * Get the information from the - * user-mode component. - * component. This transaction will be - * completed when we get the value from - * the user-mode component. - * Set a timeout to deal with - * user-mode not responding. - */ - schedule_work(&kvp_sendkey_work); - schedule_delayed_work(&kvp_timeout_work, - HV_UTIL_TIMEOUT * HZ); + /* + * Stash away this global state for completing the + * transaction; note transactions are serialized. + */ - return; + kvp_transaction.recv_len = recvlen; + kvp_transaction.recv_req_id = requestid; + kvp_transaction.kvp_msg = kvp_msg; + if (kvp_transaction.state < HVUTIL_READY) { + /* Userspace is not registered yet */ + kvp_respond_to_host(NULL, HV_E_FAIL); + return; } + kvp_transaction.state = HVUTIL_HOSTMSG_RECEIVED; - icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION - | ICMSGHDRFLAG_RESPONSE; + /* + * Get the information from the + * user-mode component. + * component. This transaction will be + * completed when we get the value from + * the user-mode component. + * Set a timeout to deal with + * user-mode not responding. + */ + schedule_work(&kvp_sendkey_work); + schedule_delayed_work(&kvp_timeout_work, + HV_UTIL_TIMEOUT * HZ); - vmbus_sendpacket(channel, recv_buffer, - recvlen, requestid, - VM_PKT_DATA_INBAND, 0); + return; - host_negotiatied = NEGO_FINISHED; - hv_poll_channel(kvp_transaction.recv_channel, kvp_poll_wrapper); + } else { + pr_err_ratelimited("KVP request received. Invalid msg type: %d\n", + icmsghdrp->icmsgtype); + return; } + icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION + | ICMSGHDRFLAG_RESPONSE; + + vmbus_sendpacket(channel, recv_buffer, + recvlen, requestid, + VM_PKT_DATA_INBAND, 0); + + host_negotiatied = NEGO_FINISHED; + hv_poll_channel(kvp_transaction.recv_channel, kvp_poll_wrapper); } static void kvp_on_reset(void) diff --git a/drivers/hv/hv_snapshot.c b/drivers/hv/hv_snapshot.c index 783779e4cc1a..2267bd4c3472 100644 --- a/drivers/hv/hv_snapshot.c +++ b/drivers/hv/hv_snapshot.c @@ -298,49 +298,64 @@ void hv_vss_onchannelcallback(void *context) if (vss_transaction.state > HVUTIL_READY) return; - vmbus_recvpacket(channel, recv_buffer, HV_HYP_PAGE_SIZE * 2, &recvlen, - &requestid); - - if (recvlen > 0) { - icmsghdrp = (struct icmsg_hdr *)&recv_buffer[ - sizeof(struct vmbuspipe_hdr)]; - - if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { - if (vmbus_prep_negotiate_resp(icmsghdrp, - recv_buffer, fw_versions, FW_VER_COUNT, - vss_versions, VSS_VER_COUNT, - NULL, &vss_srv_version)) { - - pr_info("VSS IC version %d.%d\n", - vss_srv_version >> 16, - vss_srv_version & 0xFFFF); - } - } else { - vss_msg = (struct hv_vss_msg *)&recv_buffer[ - sizeof(struct vmbuspipe_hdr) + - sizeof(struct icmsg_hdr)]; - - /* - * Stash away this global state for completing the - * transaction; note transactions are serialized. - */ - - vss_transaction.recv_len = recvlen; - vss_transaction.recv_req_id = requestid; - vss_transaction.msg = (struct hv_vss_msg *)vss_msg; - - schedule_work(&vss_handle_request_work); + if (vmbus_recvpacket(channel, recv_buffer, HV_HYP_PAGE_SIZE * 2, &recvlen, &requestid)) { + pr_err_ratelimited("VSS request received. Could not read into recv buf\n"); + return; + } + + if (!recvlen) + return; + + /* Ensure recvlen is big enough to read header data */ + if (recvlen < ICMSG_HDR) { + pr_err_ratelimited("VSS request received. Packet length too small: %d\n", + recvlen); + return; + } + + icmsghdrp = (struct icmsg_hdr *)&recv_buffer[sizeof(struct vmbuspipe_hdr)]; + + if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { + if (vmbus_prep_negotiate_resp(icmsghdrp, + recv_buffer, recvlen, + fw_versions, FW_VER_COUNT, + vss_versions, VSS_VER_COUNT, + NULL, &vss_srv_version)) { + + pr_info("VSS IC version %d.%d\n", + vss_srv_version >> 16, + vss_srv_version & 0xFFFF); + } + } else if (icmsghdrp->icmsgtype == ICMSGTYPE_VSS) { + /* Ensure recvlen is big enough to contain hv_vss_msg */ + if (recvlen < ICMSG_HDR + sizeof(struct hv_vss_msg)) { + pr_err_ratelimited("Invalid VSS msg. Packet length too small: %u\n", + recvlen); return; } + vss_msg = (struct hv_vss_msg *)&recv_buffer[ICMSG_HDR]; + + /* + * Stash away this global state for completing the + * transaction; note transactions are serialized. + */ - icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION - | ICMSGHDRFLAG_RESPONSE; + vss_transaction.recv_len = recvlen; + vss_transaction.recv_req_id = requestid; + vss_transaction.msg = (struct hv_vss_msg *)vss_msg; - vmbus_sendpacket(channel, recv_buffer, - recvlen, requestid, - VM_PKT_DATA_INBAND, 0); + schedule_work(&vss_handle_request_work); + return; + } else { + pr_err_ratelimited("VSS request received. Invalid msg type: %d\n", + icmsghdrp->icmsgtype); + return; } + icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION | + ICMSGHDRFLAG_RESPONSE; + vmbus_sendpacket(channel, recv_buffer, recvlen, requestid, + VM_PKT_DATA_INBAND, 0); } static void vss_on_reset(void) diff --git a/drivers/hv/hv_util.c b/drivers/hv/hv_util.c index 05566ecdbe4b..34f3e789cc9a 100644 --- a/drivers/hv/hv_util.c +++ b/drivers/hv/hv_util.c @@ -195,73 +195,88 @@ static void shutdown_onchannelcallback(void *context) struct icmsg_hdr *icmsghdrp; - vmbus_recvpacket(channel, shut_txf_buf, - HV_HYP_PAGE_SIZE, &recvlen, &requestid); + if (vmbus_recvpacket(channel, shut_txf_buf, HV_HYP_PAGE_SIZE, &recvlen, &requestid)) { + pr_err_ratelimited("Shutdown request received. Could not read into shut txf buf\n"); + return; + } - if (recvlen > 0) { - icmsghdrp = (struct icmsg_hdr *)&shut_txf_buf[ - sizeof(struct vmbuspipe_hdr)]; + if (!recvlen) + return; - if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { - if (vmbus_prep_negotiate_resp(icmsghdrp, shut_txf_buf, - fw_versions, FW_VER_COUNT, - sd_versions, SD_VER_COUNT, - NULL, &sd_srv_version)) { - pr_info("Shutdown IC version %d.%d\n", - sd_srv_version >> 16, - sd_srv_version & 0xFFFF); - } - } else { - shutdown_msg = - (struct shutdown_msg_data *)&shut_txf_buf[ - sizeof(struct vmbuspipe_hdr) + - sizeof(struct icmsg_hdr)]; + /* Ensure recvlen is big enough to read header data */ + if (recvlen < ICMSG_HDR) { + pr_err_ratelimited("Shutdown request received. Packet length too small: %d\n", + recvlen); + return; + } - /* - * shutdown_msg->flags can be 0(shut down), 2(reboot), - * or 4(hibernate). It may bitwise-OR 1, which means - * performing the request by force. Linux always tries - * to perform the request by force. - */ - switch (shutdown_msg->flags) { - case 0: - case 1: - icmsghdrp->status = HV_S_OK; - work = &shutdown_work; - pr_info("Shutdown request received -" - " graceful shutdown initiated\n"); - break; - case 2: - case 3: - icmsghdrp->status = HV_S_OK; - work = &restart_work; - pr_info("Restart request received -" - " graceful restart initiated\n"); - break; - case 4: - case 5: - pr_info("Hibernation request received\n"); - icmsghdrp->status = hibernation_supported ? - HV_S_OK : HV_E_FAIL; - if (hibernation_supported) - work = &hibernate_context.work; - break; - default: - icmsghdrp->status = HV_E_FAIL; - pr_info("Shutdown request received -" - " Invalid request\n"); - break; - } + icmsghdrp = (struct icmsg_hdr *)&shut_txf_buf[sizeof(struct vmbuspipe_hdr)]; + + if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { + if (vmbus_prep_negotiate_resp(icmsghdrp, + shut_txf_buf, recvlen, + fw_versions, FW_VER_COUNT, + sd_versions, SD_VER_COUNT, + NULL, &sd_srv_version)) { + pr_info("Shutdown IC version %d.%d\n", + sd_srv_version >> 16, + sd_srv_version & 0xFFFF); + } + } else if (icmsghdrp->icmsgtype == ICMSGTYPE_SHUTDOWN) { + /* Ensure recvlen is big enough to contain shutdown_msg_data struct */ + if (recvlen < ICMSG_HDR + sizeof(struct shutdown_msg_data)) { + pr_err_ratelimited("Invalid shutdown msg data. Packet length too small: %u\n", + recvlen); + return; } - icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION - | ICMSGHDRFLAG_RESPONSE; - - vmbus_sendpacket(channel, shut_txf_buf, - recvlen, requestid, - VM_PKT_DATA_INBAND, 0); + shutdown_msg = (struct shutdown_msg_data *)&shut_txf_buf[ICMSG_HDR]; + + /* + * shutdown_msg->flags can be 0(shut down), 2(reboot), + * or 4(hibernate). It may bitwise-OR 1, which means + * performing the request by force. Linux always tries + * to perform the request by force. + */ + switch (shutdown_msg->flags) { + case 0: + case 1: + icmsghdrp->status = HV_S_OK; + work = &shutdown_work; + pr_info("Shutdown request received - graceful shutdown initiated\n"); + break; + case 2: + case 3: + icmsghdrp->status = HV_S_OK; + work = &restart_work; + pr_info("Restart request received - graceful restart initiated\n"); + break; + case 4: + case 5: + pr_info("Hibernation request received\n"); + icmsghdrp->status = hibernation_supported ? + HV_S_OK : HV_E_FAIL; + if (hibernation_supported) + work = &hibernate_context.work; + break; + default: + icmsghdrp->status = HV_E_FAIL; + pr_info("Shutdown request received - Invalid request\n"); + break; + } + } else { + icmsghdrp->status = HV_E_FAIL; + pr_err_ratelimited("Shutdown request received. Invalid msg type: %d\n", + icmsghdrp->icmsgtype); } + icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION + | ICMSGHDRFLAG_RESPONSE; + + vmbus_sendpacket(channel, shut_txf_buf, + recvlen, requestid, + VM_PKT_DATA_INBAND, 0); + if (work) schedule_work(work); } @@ -396,19 +411,27 @@ static void timesync_onchannelcallback(void *context) HV_HYP_PAGE_SIZE, &recvlen, &requestid); if (ret) { - pr_warn_once("TimeSync IC pkt recv failed (Err: %d)\n", - ret); + pr_err_ratelimited("TimeSync IC pkt recv failed (Err: %d)\n", + ret); break; } if (!recvlen) break; + /* Ensure recvlen is big enough to read header data */ + if (recvlen < ICMSG_HDR) { + pr_err_ratelimited("Timesync request received. Packet length too small: %d\n", + recvlen); + break; + } + icmsghdrp = (struct icmsg_hdr *)&time_txf_buf[ sizeof(struct vmbuspipe_hdr)]; if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { - if (vmbus_prep_negotiate_resp(icmsghdrp, time_txf_buf, + if (vmbus_prep_negotiate_resp(icmsghdrp, + time_txf_buf, recvlen, fw_versions, FW_VER_COUNT, ts_versions, TS_VER_COUNT, NULL, &ts_srv_version)) { @@ -416,33 +439,44 @@ static void timesync_onchannelcallback(void *context) ts_srv_version >> 16, ts_srv_version & 0xFFFF); } - } else { + } else if (icmsghdrp->icmsgtype == ICMSGTYPE_TIMESYNC) { if (ts_srv_version > TS_VERSION_3) { - refdata = (struct ictimesync_ref_data *) - &time_txf_buf[ - sizeof(struct vmbuspipe_hdr) + - sizeof(struct icmsg_hdr)]; + /* Ensure recvlen is big enough to read ictimesync_ref_data */ + if (recvlen < ICMSG_HDR + sizeof(struct ictimesync_ref_data)) { + pr_err_ratelimited("Invalid ictimesync ref data. Length too small: %u\n", + recvlen); + break; + } + refdata = (struct ictimesync_ref_data *)&time_txf_buf[ICMSG_HDR]; adj_guesttime(refdata->parenttime, refdata->vmreferencetime, refdata->flags); } else { - timedatap = (struct ictimesync_data *) - &time_txf_buf[ - sizeof(struct vmbuspipe_hdr) + - sizeof(struct icmsg_hdr)]; + /* Ensure recvlen is big enough to read ictimesync_data */ + if (recvlen < ICMSG_HDR + sizeof(struct ictimesync_data)) { + pr_err_ratelimited("Invalid ictimesync data. Length too small: %u\n", + recvlen); + break; + } + timedatap = (struct ictimesync_data *)&time_txf_buf[ICMSG_HDR]; + adj_guesttime(timedatap->parenttime, hv_read_reference_counter(), timedatap->flags); } + } else { + icmsghdrp->status = HV_E_FAIL; + pr_err_ratelimited("Timesync request received. Invalid msg type: %d\n", + icmsghdrp->icmsgtype); } icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION | ICMSGHDRFLAG_RESPONSE; vmbus_sendpacket(channel, time_txf_buf, - recvlen, requestid, - VM_PKT_DATA_INBAND, 0); + recvlen, requestid, + VM_PKT_DATA_INBAND, 0); } } @@ -462,18 +496,28 @@ static void heartbeat_onchannelcallback(void *context) while (1) { - vmbus_recvpacket(channel, hbeat_txf_buf, - HV_HYP_PAGE_SIZE, &recvlen, &requestid); + if (vmbus_recvpacket(channel, hbeat_txf_buf, HV_HYP_PAGE_SIZE, + &recvlen, &requestid)) { + pr_err_ratelimited("Heartbeat request received. Could not read into hbeat txf buf\n"); + return; + } if (!recvlen) break; + /* Ensure recvlen is big enough to read header data */ + if (recvlen < ICMSG_HDR) { + pr_err_ratelimited("Hearbeat request received. Packet length too small: %d\n", + recvlen); + break; + } + icmsghdrp = (struct icmsg_hdr *)&hbeat_txf_buf[ sizeof(struct vmbuspipe_hdr)]; if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { if (vmbus_prep_negotiate_resp(icmsghdrp, - hbeat_txf_buf, + hbeat_txf_buf, recvlen, fw_versions, FW_VER_COUNT, hb_versions, HB_VER_COUNT, NULL, &hb_srv_version)) { @@ -482,21 +526,31 @@ static void heartbeat_onchannelcallback(void *context) hb_srv_version >> 16, hb_srv_version & 0xFFFF); } - } else { - heartbeat_msg = - (struct heartbeat_msg_data *)&hbeat_txf_buf[ - sizeof(struct vmbuspipe_hdr) + - sizeof(struct icmsg_hdr)]; + } else if (icmsghdrp->icmsgtype == ICMSGTYPE_HEARTBEAT) { + /* + * Ensure recvlen is big enough to read seq_num. Reserved area is not + * included in the check as the host may not fill it up entirely + */ + if (recvlen < ICMSG_HDR + sizeof(u64)) { + pr_err_ratelimited("Invalid heartbeat msg data. Length too small: %u\n", + recvlen); + break; + } + heartbeat_msg = (struct heartbeat_msg_data *)&hbeat_txf_buf[ICMSG_HDR]; heartbeat_msg->seq_num += 1; + } else { + icmsghdrp->status = HV_E_FAIL; + pr_err_ratelimited("Heartbeat request received. Invalid msg type: %d\n", + icmsghdrp->icmsgtype); } icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION | ICMSGHDRFLAG_RESPONSE; vmbus_sendpacket(channel, hbeat_txf_buf, - recvlen, requestid, - VM_PKT_DATA_INBAND, 0); + recvlen, requestid, + VM_PKT_DATA_INBAND, 0); } } diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index fbae8406d5d4..2ea967bc17ad 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -1480,6 +1480,7 @@ void vmbus_free_mmio(resource_size_t start, resource_size_t size); #define ICMSGTYPE_SHUTDOWN 3 #define ICMSGTYPE_TIMESYNC 4 #define ICMSGTYPE_VSS 5 +#define ICMSGTYPE_FCOPY 7 #define ICMSGHDRFLAG_TRANSACTION 1 #define ICMSGHDRFLAG_REQUEST 2 @@ -1523,6 +1524,12 @@ struct icmsg_hdr { u8 reserved[2]; } __packed; +#define IC_VERSION_NEGOTIATION_MAX_VER_COUNT 100 +#define ICMSG_HDR (sizeof(struct vmbuspipe_hdr) + sizeof(struct icmsg_hdr)) +#define ICMSG_NEGOTIATE_PKT_SIZE(icframe_vercnt, icmsg_vercnt) \ + (ICMSG_HDR + offsetof(struct icmsg_negotiate, icversion_data) + \ + (((icframe_vercnt) + (icmsg_vercnt)) * sizeof(struct ic_version))) + struct icmsg_negotiate { u16 icframe_vercnt; u16 icmsg_vercnt; @@ -1578,7 +1585,7 @@ struct hyperv_service_callback { }; #define MAX_SRV_VER 0x7ffffff -extern bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf, +extern bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf, u32 buflen, const int *fw_version, int fw_vercnt, const int *srv_version, int srv_vercnt, int *nego_fw_version, int *nego_srv_version); -- cgit v1.2.3 From e4d221b42354b2e2ddb9187a806afb651eee2cda Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Wed, 9 Dec 2020 08:08:26 +0100 Subject: Drivers: hv: vmbus: Resolve race condition in vmbus_onoffer_rescind() An erroneous or malicious host could send multiple rescind messages for a same channel. In vmbus_onoffer_rescind(), the guest maps the channel ID to obtain a pointer to the channel object and it eventually releases such object and associated data. The host could time rescind messages and lead to an use-after-free. Add a new flag to the channel structure to make sure that only one instance of vmbus_onoffer_rescind() can get the reference to the channel object. Reported-by: Juan Vazquez Signed-off-by: Andrea Parri (Microsoft) Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20201209070827.29335-6-parri.andrea@gmail.com Signed-off-by: Wei Liu --- drivers/hv/channel_mgmt.c | 12 ++++++++++++ include/linux/hyperv.h | 1 + 2 files changed, 13 insertions(+) (limited to 'include/linux') diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 4072fd1f2214..68950a1e4b63 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -1063,6 +1063,18 @@ static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr) mutex_lock(&vmbus_connection.channel_mutex); channel = relid2channel(rescind->child_relid); + if (channel != NULL) { + /* + * Guarantee that no other instance of vmbus_onoffer_rescind() + * has got a reference to the channel object. Synchronize on + * &vmbus_connection.channel_mutex. + */ + if (channel->rescind_ref) { + mutex_unlock(&vmbus_connection.channel_mutex); + return; + } + channel->rescind_ref = true; + } mutex_unlock(&vmbus_connection.channel_mutex); if (channel == NULL) { diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 2ea967bc17ad..f0d48a368f13 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -809,6 +809,7 @@ struct vmbus_channel { u8 monitor_bit; bool rescind; /* got rescind msg */ + bool rescind_ref; /* got rescind msg, got channel reference */ struct completion rescind_event; u32 ringbuffer_gpadlhandle; -- cgit v1.2.3 From 585fc0d2871c9318c949fbf45b1f081edd489e96 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 4 Feb 2021 18:32:03 -0800 Subject: mm: hugetlbfs: fix cannot migrate the fallocated HugeTLB page If a new hugetlb page is allocated during fallocate it will not be marked as active (set_page_huge_active) which will result in a later isolate_huge_page failure when the page migration code would like to move that page. Such a failure would be unexpected and wrong. Only export set_page_huge_active, just leave clear_page_huge_active as static. Because there are no external users. Link: https://lkml.kernel.org/r/20210115124942.46403-3-songmuchun@bytedance.com Fixes: 70c3547e36f5 (hugetlbfs: add hugetlbfs_fallocate()) Signed-off-by: Muchun Song Acked-by: Michal Hocko Reviewed-by: Mike Kravetz Reviewed-by: Oscar Salvador Cc: David Hildenbrand Cc: Yang Shi Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/hugetlbfs/inode.c | 3 ++- include/linux/hugetlb.h | 2 ++ mm/hugetlb.c | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index b5c109703daa..21c20fd5f9ee 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -735,9 +735,10 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, mutex_unlock(&hugetlb_fault_mutex_table[hash]); + set_page_huge_active(page); /* * unlock_page because locked by add_to_page_cache() - * page_put due to reference from alloc_huge_page() + * put_page() due to reference from alloc_huge_page() */ unlock_page(page); put_page(page); diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index ebca2ef02212..b5807f23caf8 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -770,6 +770,8 @@ static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, } #endif +void set_page_huge_active(struct page *page); + #else /* CONFIG_HUGETLB_PAGE */ struct hstate {}; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 18f6ee317900..6f0e242d38ca 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1349,7 +1349,7 @@ bool page_huge_active(struct page *page) } /* never called for tail page */ -static void set_page_huge_active(struct page *page) +void set_page_huge_active(struct page *page) { VM_BUG_ON_PAGE(!PageHeadHuge(page), page); SetPagePrivate(&page[1]); -- cgit v1.2.3 From 4f6ec8602341e97b364e4e0d41a1ed08148f5e98 Mon Sep 17 00:00:00 2001 From: Rick Edgecombe Date: Thu, 4 Feb 2021 18:32:24 -0800 Subject: mm/vmalloc: separate put pages and flush VM flags When VM_MAP_PUT_PAGES was added, it was defined with the same value as VM_FLUSH_RESET_PERMS. This doesn't seem like it will cause any big functional problems other than some excess flushing for VM_MAP_PUT_PAGES allocations. Redefine VM_MAP_PUT_PAGES to have its own value. Also, rearrange things so flags are less likely to be missed in the future. Link: https://lkml.kernel.org/r/20210122233706.9304-1-rick.p.edgecombe@intel.com Fixes: b944afc9d64d ("mm: add a VM_MAP_PUT_PAGES flag for vmap") Signed-off-by: Rick Edgecombe Suggested-by: Matthew Wilcox Cc: Miaohe Lin Cc: Christoph Hellwig Cc: Daniel Axtens Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/vmalloc.h | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 80c0181c411d..cedcda6593f6 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -24,7 +24,8 @@ struct notifier_block; /* in notifier.h */ #define VM_UNINITIALIZED 0x00000020 /* vm_struct is not fully initialized */ #define VM_NO_GUARD 0x00000040 /* don't add guard page */ #define VM_KASAN 0x00000080 /* has allocated kasan shadow memory */ -#define VM_MAP_PUT_PAGES 0x00000100 /* put pages and free array in vfree */ +#define VM_FLUSH_RESET_PERMS 0x00000100 /* reset direct map and flush TLB on unmap, can't be freed in atomic context */ +#define VM_MAP_PUT_PAGES 0x00000200 /* put pages and free array in vfree */ /* * VM_KASAN is used slighly differently depending on CONFIG_KASAN_VMALLOC. @@ -37,12 +38,6 @@ struct notifier_block; /* in notifier.h */ * determine which allocations need the module shadow freed. */ -/* - * Memory with VM_FLUSH_RESET_PERMS cannot be freed in an interrupt or with - * vfree_atomic(). - */ -#define VM_FLUSH_RESET_PERMS 0x00000100 /* Reset direct map and flush TLB on unmap */ - /* bits [20..32] reserved for arch specific ioremap internals */ /* -- cgit v1.2.3 From 49c6631d3b4f61a7b5bb0453a885a12bfa06ffd8 Mon Sep 17 00:00:00 2001 From: Vincenzo Frascino Date: Thu, 4 Feb 2021 18:32:49 -0800 Subject: kasan: add explicit preconditions to kasan_report() Patch series "kasan: Fix metadata detection for KASAN_HW_TAGS", v5. With the introduction of KASAN_HW_TAGS, kasan_report() currently assumes that every location in memory has valid metadata associated. This is due to the fact that addr_has_metadata() returns always true. As a consequence of this, an invalid address (e.g. NULL pointer address) passed to kasan_report() when KASAN_HW_TAGS is enabled, leads to a kernel panic. Example below, based on arm64: BUG: KASAN: invalid-access in 0x0 Read at addr 0000000000000000 by task swapper/0/1 Unable to handle kernel NULL pointer dereference at virtual address 0000000000000000 Mem abort info: ESR = 0x96000004 EC = 0x25: DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 Data abort info: ISV = 0, ISS = 0x00000004 CM = 0, WnR = 0 ... Call trace: mte_get_mem_tag+0x24/0x40 kasan_report+0x1a4/0x410 alsa_sound_last_init+0x8c/0xa4 do_one_initcall+0x50/0x1b0 kernel_init_freeable+0x1d4/0x23c kernel_init+0x14/0x118 ret_from_fork+0x10/0x34 Code: d65f03c0 9000f021 f9428021 b6cfff61 (d9600000) ---[ end trace 377c8bb45bdd3a1a ]--- hrtimer: interrupt took 48694256 ns note: swapper/0[1] exited with preempt_count 1 Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b SMP: stopping secondary CPUs Kernel Offset: 0x35abaf140000 from 0xffff800010000000 PHYS_OFFSET: 0x40000000 CPU features: 0x0a7e0152,61c0a030 Memory Limit: none ---[ end Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b ]--- This series fixes the behavior of addr_has_metadata() that now returns true only when the address is valid. This patch (of 2): With the introduction of KASAN_HW_TAGS, kasan_report() accesses the metadata only when addr_has_metadata() succeeds. Add a comment to make sure that the preconditions to the function are explicitly clarified. Link: https://lkml.kernel.org/r/20210126134409.47894-1-vincenzo.frascino@arm.com Link: https://lkml.kernel.org/r/20210126134409.47894-2-vincenzo.frascino@arm.com Signed-off-by: Vincenzo Frascino Reviewed-by: Andrey Konovalov Cc: Andrey Ryabinin Cc: Alexander Potapenko Cc: Dmitry Vyukov Cc: Leon Romanovsky Cc: Andrey Konovalov Cc: Catalin Marinas Cc: Will Deacon Cc: Mark Rutland Cc: "Paul E . McKenney" Cc: Naresh Kamboju Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/kasan.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/linux') diff --git a/include/linux/kasan.h b/include/linux/kasan.h index fe1ae73ff8b5..0aea9e2a2a01 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -333,6 +333,13 @@ static inline void *kasan_reset_tag(const void *addr) return (void *)arch_kasan_reset_tag(addr); } +/** + * kasan_report - print a report about a bad memory access detected by KASAN + * @addr: address of the bad access + * @size: size of the bad access + * @is_write: whether the bad access is a write or a read + * @ip: instruction pointer for the accessibility check or the bad access itself + */ bool kasan_report(unsigned long addr, size_t size, bool is_write, unsigned long ip); -- cgit v1.2.3 From 4c7bcb51ae25f79e3733982e5d0cd8ce8640ddfc Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Mon, 21 Dec 2020 19:56:47 +0100 Subject: genirq: Prevent [devm_]irq_alloc_desc from returning irq 0 Since commit a85a6c86c25b ("driver core: platform: Clarify that IRQ 0 is invalid"), having a linux-irq with number 0 will trigger a WARN() when calling platform_get_irq*() to retrieve that linux-irq. Since [devm_]irq_alloc_desc allocs a single irq and since irq 0 is not used on some systems, it can return 0, triggering that WARN(). This happens e.g. on Intel Bay Trail and Cherry Trail devices using the LPE audio engine for HDMI audio: 0 is an invalid IRQ number WARNING: CPU: 3 PID: 472 at drivers/base/platform.c:238 platform_get_irq_optional+0x108/0x180 Modules linked in: snd_hdmi_lpe_audio(+) ... Call Trace: platform_get_irq+0x17/0x30 hdmi_lpe_audio_probe+0x4a/0x6c0 [snd_hdmi_lpe_audio] ---[ end trace ceece38854223a0b ]--- Change the 'from' parameter passed to __[devm_]irq_alloc_descs() by the [devm_]irq_alloc_desc macros from 0 to 1, so that these macros will no longer return 0. Fixes: a85a6c86c25b ("driver core: platform: Clarify that IRQ 0 is invalid") Signed-off-by: Hans de Goede Signed-off-by: Thomas Gleixner Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20201221185647.226146-1-hdegoede@redhat.com --- include/linux/irq.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/irq.h b/include/linux/irq.h index 4aeb1c4c7e07..2efde6a79b7e 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -928,7 +928,7 @@ int __devm_irq_alloc_descs(struct device *dev, int irq, unsigned int from, __irq_alloc_descs(irq, from, cnt, node, THIS_MODULE, NULL) #define irq_alloc_desc(node) \ - irq_alloc_descs(-1, 0, 1, node) + irq_alloc_descs(-1, 1, 1, node) #define irq_alloc_desc_at(at, node) \ irq_alloc_descs(at, at, 1, node) @@ -943,7 +943,7 @@ int __devm_irq_alloc_descs(struct device *dev, int irq, unsigned int from, __devm_irq_alloc_descs(dev, irq, from, cnt, node, THIS_MODULE, NULL) #define devm_irq_alloc_desc(dev, node) \ - devm_irq_alloc_descs(dev, -1, 0, 1, node) + devm_irq_alloc_descs(dev, -1, 1, 1, node) #define devm_irq_alloc_desc_at(dev, at, node) \ devm_irq_alloc_descs(dev, at, at, 1, node) -- cgit v1.2.3 From 6342adcaa683c2b705c24ed201dc11b35854c88d Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Wed, 3 Feb 2021 13:00:48 -0500 Subject: entry: Ensure trap after single-step on system call return Commit 299155244770 ("entry: Drop usage of TIF flags in the generic syscall code") introduced a bug on architectures using the generic syscall entry code, in which processes stopped by PTRACE_SYSCALL do not trap on syscall return after receiving a TIF_SINGLESTEP. The reason is that the meaning of TIF_SINGLESTEP flag is overloaded to cause the trap after a system call is executed, but since the above commit, the syscall call handler only checks for the SYSCALL_WORK flags on the exit work. Split the meaning of TIF_SINGLESTEP such that it only means single-step mode, and create a new type of SYSCALL_WORK to request a trap immediately after a syscall in single-step mode. In the current implementation, the SYSCALL_WORK flag shadows the TIF_SINGLESTEP flag for simplicity. Update x86 to flip this bit when a tracer enables single stepping. Fixes: 299155244770 ("entry: Drop usage of TIF flags in the generic syscall code") Suggested-by: Linus Torvalds Signed-off-by: Gabriel Krisman Bertazi Signed-off-by: Thomas Gleixner Tested-by: Kyle Huey Link: https://lore.kernel.org/r/87h7mtc9pr.fsf_-_@collabora.com --- arch/x86/include/asm/entry-common.h | 2 -- arch/x86/kernel/step.c | 10 ++++++++-- include/linux/entry-common.h | 1 + include/linux/thread_info.h | 2 ++ kernel/entry/common.c | 12 ++---------- 5 files changed, 13 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h index 6fe54b2813c1..2b87b191b3b8 100644 --- a/arch/x86/include/asm/entry-common.h +++ b/arch/x86/include/asm/entry-common.h @@ -43,8 +43,6 @@ static __always_inline void arch_check_user_regs(struct pt_regs *regs) } #define arch_check_user_regs arch_check_user_regs -#define ARCH_SYSCALL_EXIT_WORK (_TIF_SINGLESTEP) - static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, unsigned long ti_work) { diff --git a/arch/x86/kernel/step.c b/arch/x86/kernel/step.c index 60d2c3798ba2..0f3c307b37b3 100644 --- a/arch/x86/kernel/step.c +++ b/arch/x86/kernel/step.c @@ -127,12 +127,17 @@ static int enable_single_step(struct task_struct *child) regs->flags |= X86_EFLAGS_TF; /* - * Always set TIF_SINGLESTEP - this guarantees that - * we single-step system calls etc.. This will also + * Always set TIF_SINGLESTEP. This will also * cause us to set TF when returning to user mode. */ set_tsk_thread_flag(child, TIF_SINGLESTEP); + /* + * Ensure that a trap is triggered once stepping out of a system + * call prior to executing any user instruction. + */ + set_task_syscall_work(child, SYSCALL_EXIT_TRAP); + oflags = regs->flags; /* Set TF on the kernel stack.. */ @@ -230,6 +235,7 @@ void user_disable_single_step(struct task_struct *child) /* Always clear TIF_SINGLESTEP... */ clear_tsk_thread_flag(child, TIF_SINGLESTEP); + clear_task_syscall_work(child, SYSCALL_EXIT_TRAP); /* But touch TF only if it was set by us.. */ if (test_and_clear_tsk_thread_flag(child, TIF_FORCED_TF)) diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h index ca86a00abe86..a104b298019a 100644 --- a/include/linux/entry-common.h +++ b/include/linux/entry-common.h @@ -46,6 +46,7 @@ SYSCALL_WORK_SYSCALL_TRACE | \ SYSCALL_WORK_SYSCALL_AUDIT | \ SYSCALL_WORK_SYSCALL_USER_DISPATCH | \ + SYSCALL_WORK_SYSCALL_EXIT_TRAP | \ ARCH_SYSCALL_WORK_EXIT) /* diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h index c8a974cead73..9b2158c69275 100644 --- a/include/linux/thread_info.h +++ b/include/linux/thread_info.h @@ -43,6 +43,7 @@ enum syscall_work_bit { SYSCALL_WORK_BIT_SYSCALL_EMU, SYSCALL_WORK_BIT_SYSCALL_AUDIT, SYSCALL_WORK_BIT_SYSCALL_USER_DISPATCH, + SYSCALL_WORK_BIT_SYSCALL_EXIT_TRAP, }; #define SYSCALL_WORK_SECCOMP BIT(SYSCALL_WORK_BIT_SECCOMP) @@ -51,6 +52,7 @@ enum syscall_work_bit { #define SYSCALL_WORK_SYSCALL_EMU BIT(SYSCALL_WORK_BIT_SYSCALL_EMU) #define SYSCALL_WORK_SYSCALL_AUDIT BIT(SYSCALL_WORK_BIT_SYSCALL_AUDIT) #define SYSCALL_WORK_SYSCALL_USER_DISPATCH BIT(SYSCALL_WORK_BIT_SYSCALL_USER_DISPATCH) +#define SYSCALL_WORK_SYSCALL_EXIT_TRAP BIT(SYSCALL_WORK_BIT_SYSCALL_EXIT_TRAP) #endif #include diff --git a/kernel/entry/common.c b/kernel/entry/common.c index 6dd82be60df8..f9d491b17b78 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -209,15 +209,9 @@ static void exit_to_user_mode_prepare(struct pt_regs *regs) lockdep_sys_exit(); } -#ifndef _TIF_SINGLESTEP -static inline bool report_single_step(unsigned long work) -{ - return false; -} -#else /* * If SYSCALL_EMU is set, then the only reason to report is when - * TIF_SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall + * SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall * instruction has been already reported in syscall_enter_from_user_mode(). */ static inline bool report_single_step(unsigned long work) @@ -225,10 +219,8 @@ static inline bool report_single_step(unsigned long work) if (work & SYSCALL_WORK_SYSCALL_EMU) return false; - return !!(current_thread_info()->flags & _TIF_SINGLESTEP); + return work & SYSCALL_WORK_SYSCALL_EXIT_TRAP; } -#endif - static void syscall_exit_work(struct pt_regs *regs, unsigned long work) { -- cgit v1.2.3 From 10742efc20a429b2040658af685d6bb2aa674a73 Mon Sep 17 00:00:00 2001 From: Vlad Buslov Date: Thu, 21 Jan 2021 19:41:52 +0200 Subject: net/mlx5e: VF tunnel TX traffic offloading When tunnel endpoint is on VF, driver still assumes that endpoint is on uplink and incorrectly configures encap rule offload according to that assumption. As a result, traffic is sent directly to the uplink and rules installed on representor of tunnel endpoint VF are ignored. Implement following changes to allow offloading tx traffic with tunnel endpoint on VF: - For tunneling flows perform route lookup on route and out devices pair. If out device is uplink and route device is VF of same physical port, then modify packet reg_c_0 metadata register (source port) with the value of VF vport. Use eswitch vhca_id->vport mapping introduced in one of previous patches in the series to obtain vport from route netdevice. - Recirculate encapsulated packets to VF vport in order to apply any flow rules installed on VF representor that match on encapsulated traffic. Only enable support for this functionality when all following conditions are true: - Hardware advertises capability to preserve reg_c_0 value on packet recirculation. - Vport metadata matching is enabled. - Termination tables are to be used by the flow. Example TC rules for VF tunnel traffic: 1. Rule that redirects packets from UL to VF rep that has the tunnel endpoint IP address: $ tc -s filter show dev enp8s0f0 ingress filter protocol ip pref 4 flower chain 0 filter protocol ip pref 4 flower chain 0 handle 0x1 dst_mac 16:c9:a0:2d:69:2c src_mac 0c:42:a1:58:ab:e4 eth_type ipv4 ip_flags nofrag in_hw in_hw_count 1 action order 1: mirred (Egress Redirect to device enp8s0f0_0) stolen index 3 ref 1 bind 1 installed 377 sec used 0 sec Action statistics: Sent 114096 bytes 952 pkt (dropped 0, overlimits 0 requeues 0) Sent software 0 bytes 0 pkt Sent hardware 114096 bytes 952 pkt backlog 0b 0p requeues 0 cookie 878fa48d8c423fc08c3b6ca599b50a97 no_percpu used_hw_stats delayed 2. Rule that decapsulates the tunneled flow and redirects to destination VF representor: $ tc -s filter show dev vxlan_sys_4789 ingress filter protocol ip pref 4 flower chain 0 filter protocol ip pref 4 flower chain 0 handle 0x1 dst_mac ca:2e:a7:3f:f5:0f src_mac 0a:40:bd:30:89:99 eth_type ipv4 enc_dst_ip 7.7.7.5 enc_src_ip 7.7.7.1 enc_key_id 98 enc_dst_port 4789 enc_tos 0 ip_flags nofrag in_hw in_hw_count 1 action order 1: tunnel_key unset pipe index 2 ref 1 bind 1 installed 434 sec used 434 sec Action statistics: Sent 0 bytes 0 pkt (dropped 0, overlimits 0 requeues 0) backlog 0b 0p requeues 0 used_hw_stats delayed action order 2: mirred (Egress Redirect to device enp8s0f0_1) stolen index 4 ref 1 bind 1 installed 434 sec used 0 sec Action statistics: Sent 129936 bytes 1082 pkt (dropped 0, overlimits 0 requeues 0) Sent software 0 bytes 0 pkt Sent hardware 129936 bytes 1082 pkt backlog 0b 0p requeues 0 cookie ac17cf398c4c69e4a5b2f7aabd1b88ff no_percpu used_hw_stats delayed Co-developed-by: Dmytro Linkin Signed-off-by: Dmytro Linkin Signed-off-by: Vlad Buslov Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en_tc.c | 87 +++++++++++++++ drivers/net/ethernet/mellanox/mlx5/core/en_tc.h | 1 + drivers/net/ethernet/mellanox/mlx5/core/eswitch.h | 2 + .../ethernet/mellanox/mlx5/core/eswitch_offloads.c | 120 +++++++++++++++++++-- include/linux/mlx5/eswitch.h | 2 + 5 files changed, 201 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 280ea1e1e039..43f1508a05b5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -165,6 +165,11 @@ struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = { .moffset = 0, .mlen = 2, }, + [VPORT_TO_REG] = { + .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_0, + .moffset = 2, + .mlen = 2, + }, [TUNNEL_TO_REG] = { .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_1, .moffset = 1, @@ -1315,6 +1320,44 @@ static void remove_unready_flow(struct mlx5e_tc_flow *flow) mutex_unlock(&uplink_priv->unready_flows_lock); } +static bool same_hw_devs(struct mlx5e_priv *priv, struct mlx5e_priv *peer_priv); + +static bool mlx5e_tc_is_vf_tunnel(struct net_device *out_dev, struct net_device *route_dev) +{ + struct mlx5_core_dev *out_mdev, *route_mdev; + struct mlx5e_priv *out_priv, *route_priv; + + out_priv = netdev_priv(out_dev); + out_mdev = out_priv->mdev; + route_priv = netdev_priv(route_dev); + route_mdev = route_priv->mdev; + + if (out_mdev->coredev_type != MLX5_COREDEV_PF || + route_mdev->coredev_type != MLX5_COREDEV_VF) + return false; + + return same_hw_devs(out_priv, route_priv); +} + +static int mlx5e_tc_query_route_vport(struct net_device *out_dev, struct net_device *route_dev, + u16 *vport) +{ + struct mlx5e_priv *out_priv, *route_priv; + struct mlx5_core_dev *route_mdev; + struct mlx5_eswitch *esw; + u16 vhca_id; + int err; + + out_priv = netdev_priv(out_dev); + esw = out_priv->mdev->priv.eswitch; + route_priv = netdev_priv(route_dev); + route_mdev = route_priv->mdev; + + vhca_id = MLX5_CAP_GEN(route_mdev, vhca_id); + err = mlx5_eswitch_vhca_id_to_vport(esw, vhca_id, vport); + return err; +} + static int mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow, @@ -3700,6 +3743,45 @@ static bool is_duplicated_encap_entry(struct mlx5e_priv *priv, return false; } +static int mlx5e_set_vf_tunnel(struct mlx5_eswitch *esw, + struct mlx5_flow_attr *attr, + struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts, + struct net_device *out_dev, + int route_dev_ifindex, + int out_index) +{ + struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; + struct net_device *route_dev; + u16 vport_num; + int err = 0; + u32 data; + + route_dev = dev_get_by_index(dev_net(out_dev), route_dev_ifindex); + + if (!route_dev || route_dev->netdev_ops != &mlx5e_netdev_ops || + !mlx5e_tc_is_vf_tunnel(out_dev, route_dev)) + goto out; + + err = mlx5e_tc_query_route_vport(out_dev, route_dev, &vport_num); + if (err) + goto out; + + attr->dest_chain = 0; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + esw_attr->dests[out_index].flags |= MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE; + data = mlx5_eswitch_get_vport_metadata_for_set(esw_attr->in_mdev->priv.eswitch, + vport_num); + err = mlx5e_tc_match_to_reg_set(esw->dev, mod_hdr_acts, + MLX5_FLOW_NAMESPACE_FDB, VPORT_TO_REG, data); + if (err) + goto out; + +out: + if (route_dev) + dev_put(route_dev); + return err; +} + static int mlx5e_attach_encap(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow, struct net_device *mirred_dev, @@ -3791,6 +3873,11 @@ static int mlx5e_attach_encap(struct mlx5e_priv *priv, e->compl_result = 1; attach_flow: + err = mlx5e_set_vf_tunnel(esw, attr, &parse_attr->mod_hdr_acts, e->out_dev, + e->route_dev_ifindex, out_index); + if (err) + goto out_err; + flow->encaps[out_index].e = e; list_add(&flow->encaps[out_index].list, &e->flows); flow->encaps[out_index].index = out_index; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h index 4a2ce241522e..56d809904ea7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h @@ -167,6 +167,7 @@ void mlx5e_tc_reoffload_flows_work(struct work_struct *work); enum mlx5e_tc_attr_to_reg { CHAIN_TO_REG, + VPORT_TO_REG, TUNNEL_TO_REG, CTSTATE_TO_REG, ZONE_TO_REG, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 1a045e95bc68..1ab34751329e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -389,12 +389,14 @@ enum mlx5_flow_match_level { enum { MLX5_ESW_DEST_ENCAP = BIT(0), MLX5_ESW_DEST_ENCAP_VALID = BIT(1), + MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE = BIT(2), }; enum { MLX5_ESW_ATTR_FLAG_VLAN_HANDLED = BIT(0), MLX5_ESW_ATTR_FLAG_SLOW_PATH = BIT(1), MLX5_ESW_ATTR_FLAG_NO_IN_PORT = BIT(2), + MLX5_ESW_ATTR_FLAG_SRC_REWRITE = BIT(3), }; struct mlx5_esw_flow_attr { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 335dc83d1bb9..1b18f624e04a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -337,6 +337,65 @@ esw_setup_chain_dest(struct mlx5_flow_destination *dest, return 0; } +static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, + int from, int to) +{ + struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; + struct mlx5_fs_chains *chains = esw_chains(esw); + int i; + + for (i = from; i < to; i++) + if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) + mlx5_chains_put_table(chains, 0, 1, 0); +} + +static bool +esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr) +{ + int i; + + for (i = esw_attr->split_count; i < esw_attr->out_count; i++) + if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) + return true; + return false; +} + +static int +esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest, + struct mlx5_flow_act *flow_act, + struct mlx5_eswitch *esw, + struct mlx5_fs_chains *chains, + struct mlx5_flow_attr *attr, + int *i) +{ + struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; + int j, err; + + if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SRC_REWRITE)) + return -EOPNOTSUPP; + + for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) { + err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i); + if (err) + goto err_setup_chain; + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + flow_act->pkt_reformat = esw_attr->dests[j].pkt_reformat; + } + return 0; + +err_setup_chain: + esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j); + return err; +} + +static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw, + struct mlx5_flow_attr *attr) +{ + struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; + + esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); +} + static void esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level) { @@ -381,12 +440,18 @@ esw_setup_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, + struct mlx5_flow_spec *spec, int *i) { struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; struct mlx5_fs_chains *chains = esw_chains(esw); int err = 0; + if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) && + MLX5_CAP_GEN(esw_attr->in_mdev, reg_c_preserve) && + mlx5_eswitch_vport_match_metadata_enabled(esw)) + attr->flags |= MLX5_ESW_ATTR_FLAG_SRC_REWRITE; + if (attr->dest_ft) { esw_setup_ft_dest(dest, flow_act, attr, *i); (*i)++; @@ -397,6 +462,8 @@ esw_setup_dests(struct mlx5_flow_destination *dest, err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i); (*i)++; + } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) { + err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i); } else { *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i); } @@ -408,10 +475,15 @@ static void esw_cleanup_dests(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) { + struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; struct mlx5_fs_chains *chains = esw_chains(esw); - if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) && attr->dest_chain) - esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0); + if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH)) { + if (attr->dest_chain) + esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0); + else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) + esw_cleanup_chain_src_port_rewrite(esw, attr); + } } struct mlx5_flow_handle * @@ -448,10 +520,12 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, } } + mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr); + if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { int err; - err = esw_setup_dests(dest, &flow_act, esw, attr, &i); + err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i); if (err) { rule = ERR_PTR(err); goto err_create_goto_table; @@ -498,8 +572,6 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, goto err_esw_get; } - mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr); - if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr, &flow_act, dest, i); @@ -536,7 +608,7 @@ mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, struct mlx5_flow_table *fast_fdb; struct mlx5_flow_table *fwd_fdb; struct mlx5_flow_handle *rule; - int i; + int i, err = 0; fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0); if (IS_ERR(fast_fdb)) { @@ -554,8 +626,18 @@ mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, } flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; - for (i = 0; i < esw_attr->split_count; i++) - esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false); + for (i = 0; i < esw_attr->split_count; i++) { + if (esw_is_chain_src_port_rewrite(esw, esw_attr)) + err = esw_setup_chain_src_port_rewrite(dest, &flow_act, esw, chains, attr, + &i); + else + esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false); + + if (err) { + rule = ERR_PTR(err); + goto err_chain_src_rewrite; + } + } dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dest[i].ft = fwd_fdb; i++; @@ -570,13 +652,16 @@ mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i); - if (IS_ERR(rule)) - goto add_err; + if (IS_ERR(rule)) { + i = esw_attr->split_count; + goto err_chain_src_rewrite; + } atomic64_inc(&esw->offloads.num_flows); return rule; -add_err: +err_chain_src_rewrite: + esw_put_dest_tables_loop(esw, attr, 0, i); esw_vport_tbl_put(esw, &fwd_attr); err_get_fwd: mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); @@ -617,6 +702,7 @@ __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, if (fwd_rule) { esw_vport_tbl_put(esw, &fwd_attr); mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); + esw_put_dest_tables_loop(esw, attr, 0, esw_attr->split_count); } else { if (split) esw_vport_tbl_put(esw, &fwd_attr); @@ -3020,3 +3106,15 @@ int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vp *vport_num = *res; return 0; } + +u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, + u16 vport_num) +{ + struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); + + if (WARN_ON_ONCE(IS_ERR(vport))) + return 0; + + return vport->metadata; +} +EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set); diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h index 29fd832950e0..67e341274a22 100644 --- a/include/linux/mlx5/eswitch.h +++ b/include/linux/mlx5/eswitch.h @@ -96,6 +96,8 @@ static inline u32 mlx5_eswitch_get_vport_metadata_mask(void) u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, u16 vport_num); +u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, + u16 vport_num); u8 mlx5_eswitch_mode(struct mlx5_core_dev *dev); #else /* CONFIG_MLX5_ESWITCH */ -- cgit v1.2.3 From 48d216e5596a58e3cfa6d4548343f982c5921b79 Mon Sep 17 00:00:00 2001 From: Vlad Buslov Date: Mon, 31 Aug 2020 16:18:19 +0300 Subject: net/mlx5e: Refactor reg_c1 usage Following patch in series uses reg_c1 in eswitch code. To use reg_c1 helpers in both TC and eswitch code, refactor existing helpers according to similar use case of reg_c0 and move the functionality into eswitch.h. Calculate reg mappings length from new defines to ensure that they are always in sync and only need to be changed in single place. Signed-off-by: Vlad Buslov Signed-off-by: Dmytro Linkin Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c | 4 ++-- drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h | 6 ++---- drivers/net/ethernet/mellanox/mlx5/core/en_tc.c | 4 ++-- drivers/net/ethernet/mellanox/mlx5/core/en_tc.h | 2 +- include/linux/mlx5/eswitch.h | 19 +++++++++++++++++++ 5 files changed, 26 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c index 76177f7c5ec2..14bcebd4a0b6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c @@ -651,7 +651,7 @@ bool mlx5e_rep_tc_update_skb(struct mlx5_cqe64 *cqe, tc_skb_ext->chain = chain; - zone_restore_id = reg_c1 & ZONE_RESTORE_MAX; + zone_restore_id = reg_c1 & ESW_ZONE_ID_MASK; uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); uplink_priv = &uplink_rpriv->uplink_priv; @@ -660,7 +660,7 @@ bool mlx5e_rep_tc_update_skb(struct mlx5_cqe64 *cqe, return false; } - tunnel_id = reg_c1 >> REG_MAPPING_SHIFT(TUNNEL_TO_REG); + tunnel_id = reg_c1 >> ESW_TUN_OFFSET; return mlx5e_restore_tunnel(priv, skb, tc_priv, tunnel_id); #endif /* CONFIG_NET_TC_SKB_EXT */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h index 6503b614337c..69e618d17071 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h @@ -73,7 +73,7 @@ struct mlx5_ct_attr { #define zone_restore_to_reg_ct {\ .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_1,\ .moffset = 0,\ - .mlen = 1,\ + .mlen = (ESW_ZONE_ID_BITS / 8),\ .soffset = MLX5_BYTE_OFF(fte_match_param,\ misc_parameters_2.metadata_reg_c_1) + 3,\ } @@ -81,14 +81,12 @@ struct mlx5_ct_attr { #define nic_zone_restore_to_reg_ct {\ .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_B,\ .moffset = 2,\ - .mlen = 1,\ + .mlen = (ESW_ZONE_ID_BITS / 8),\ } #define REG_MAPPING_MLEN(reg) (mlx5e_tc_attr_to_reg_mappings[reg].mlen) #define REG_MAPPING_MOFFSET(reg) (mlx5e_tc_attr_to_reg_mappings[reg].moffset) #define REG_MAPPING_SHIFT(reg) (REG_MAPPING_MOFFSET(reg) * 8) -#define ZONE_RESTORE_BITS (REG_MAPPING_MLEN(ZONE_RESTORE_TO_REG) * 8) -#define ZONE_RESTORE_MAX GENMASK(ZONE_RESTORE_BITS - 1, 0) #if IS_ENABLED(CONFIG_MLX5_TC_CT) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 098f3efa5d4d..90db5a99879d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -173,7 +173,7 @@ struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = { [TUNNEL_TO_REG] = { .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_1, .moffset = 1, - .mlen = 3, + .mlen = ((ESW_TUN_OPTS_BITS + ESW_TUN_ID_BITS) / 8), .soffset = MLX5_BYTE_OFF(fte_match_param, misc_parameters_2.metadata_reg_c_1), }, @@ -5649,7 +5649,7 @@ bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe, tc_skb_ext->chain = chain; zone_restore_id = (reg_b >> REG_MAPPING_SHIFT(NIC_ZONE_RESTORE_TO_REG)) & - ZONE_RESTORE_MAX; + ESW_ZONE_ID_MASK; if (!mlx5e_tc_ct_restore_flow(tc->ct, skb, zone_restore_id)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h index ee0029192504..1e4ee02bfb1c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h @@ -302,7 +302,7 @@ static inline bool mlx5e_cqe_regb_chain(struct mlx5_cqe64 *cqe) reg_b = be32_to_cpu(cqe->ft_metadata); - if (reg_b >> (MLX5E_TC_TABLE_CHAIN_TAG_BITS + ZONE_RESTORE_BITS)) + if (reg_b >> (MLX5E_TC_TABLE_CHAIN_TAG_BITS + ESW_ZONE_ID_BITS)) return false; chain = reg_b & MLX5E_TC_TABLE_CHAIN_TAG_MASK; diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h index 67e341274a22..3b20e84049c1 100644 --- a/include/linux/mlx5/eswitch.h +++ b/include/linux/mlx5/eswitch.h @@ -98,6 +98,25 @@ u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, u16 vport_num); u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, u16 vport_num); + +/* Reg C1 usage: + * Reg C1 = < ESW_TUN_ID(12) | ESW_TUN_OPTS(12) | ESW_ZONE_ID(8) > + * + * Highest 12 bits of reg c1 is the encapsulation tunnel id, next 12 bits is + * encapsulation tunnel options, and the lowest 8 bits are used for zone id. + * + * Zone id is used to restore CT flow when packet misses on chain. + * + * Tunnel id and options are used together to restore the tunnel info metadata + * on miss and to support inner header rewrite by means of implicit chain 0 + * flows. + */ +#define ESW_ZONE_ID_BITS 8 +#define ESW_TUN_OPTS_BITS 12 +#define ESW_TUN_ID_BITS 12 +#define ESW_TUN_OFFSET ESW_ZONE_ID_BITS +#define ESW_ZONE_ID_MASK GENMASK(ESW_ZONE_ID_BITS - 1, 0) + u8 mlx5_eswitch_mode(struct mlx5_core_dev *dev); #else /* CONFIG_MLX5_ESWITCH */ -- cgit v1.2.3 From 8e404fefa58b6138531e3d4b5647ee79f75ae9a8 Mon Sep 17 00:00:00 2001 From: Vlad Buslov Date: Mon, 31 Aug 2020 16:18:57 +0300 Subject: net/mlx5e: Match recirculated packet miss in slow table using reg_c1 Previous patch in series that implements stack devices RX path implements indirect table rules that match on tunnel VNI. After such rule is created all tunnel traffic is recirculated to root table. However, recirculated packet might not match on any rules installed in the table (for example, when IP traffic follows ARP traffic). In that case packets appear on representor of tunnel endpoint VF instead being redirected to the VF itself. Extend slow table with additional flow group that matches on reg_c0 (source port value set by indirect tables implemented by previous patch in series) and reg_c1 (special 0xFFF mark). When creating offloads fdb tables, install one rule per VF vport to match on recirculated miss packets and redirect them to appropriate VF vport. Modify indirect tables code to also rewrite reg_c1 with special 0xFFF mark. Implementation reuses reg_c1 tunnel id bits. This is safe to do because recirculated packets are always matched before decapsulation. Signed-off-by: Vlad Buslov Signed-off-by: Dmytro Linkin Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en_tc.c | 3 +- .../ethernet/mellanox/mlx5/core/esw/indir_table.c | 17 ++- drivers/net/ethernet/mellanox/mlx5/core/eswitch.h | 2 + .../ethernet/mellanox/mlx5/core/eswitch_offloads.c | 119 ++++++++++++++++++++- include/linux/mlx5/eswitch.h | 10 +- 5 files changed, 143 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 90db5a99879d..21568d1fc00f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -5515,7 +5515,8 @@ int mlx5e_tc_esw_init(struct rhashtable *tc_ht) } uplink_priv->tunnel_mapping = mapping; - mapping = mapping_create(sz_enc_opts, ENC_OPTS_BITS_MASK, true); + /* 0xFFF is reserved for stack devices slow path table mark */ + mapping = mapping_create(sz_enc_opts, ENC_OPTS_BITS_MASK - 1, true); if (IS_ERR(mapping)) { err = PTR_ERR(mapping); goto err_enc_opts_mapping; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c index a0ebf40c9907..b7d00c4c7046 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c @@ -162,7 +162,7 @@ static int mlx5_esw_indir_table_rule_get(struct mlx5_eswitch *esw, (attr->ip_version == 4 ? ETH_P_IP : ETH_P_IPV6)); } else { err = -EOPNOTSUPP; - goto err_mod_hdr; + goto err_ethertype; } if (attr->ip_version == 4) { @@ -198,13 +198,18 @@ static int mlx5_esw_indir_table_rule_get(struct mlx5_eswitch *esw, err = mlx5e_tc_match_to_reg_set(esw->dev, &mod_acts, MLX5_FLOW_NAMESPACE_FDB, VPORT_TO_REG, data); if (err) - goto err_mod_hdr; + goto err_mod_hdr_regc0; + + err = mlx5e_tc_match_to_reg_set(esw->dev, &mod_acts, MLX5_FLOW_NAMESPACE_FDB, + TUNNEL_TO_REG, ESW_TUN_SLOW_TABLE_GOTO_VPORT); + if (err) + goto err_mod_hdr_regc1; flow_act.modify_hdr = mlx5_modify_header_alloc(esw->dev, MLX5_FLOW_NAMESPACE_FDB, mod_acts.num_actions, mod_acts.actions); if (IS_ERR(flow_act.modify_hdr)) { err = PTR_ERR(flow_act.modify_hdr); - goto err_mod_hdr; + goto err_mod_hdr_alloc; } flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; @@ -236,7 +241,11 @@ err_handle: mlx5_chains_put_table(chains, 0, 1, 0); err_table: mlx5_modify_header_dealloc(esw->dev, flow_act.modify_hdr); -err_mod_hdr: +err_mod_hdr_alloc: +err_mod_hdr_regc1: + dealloc_mod_hdr_actions(&mod_acts); +err_mod_hdr_regc0: +err_ethertype: kfree(rule); out: kfree(rule_spec); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index c2361c5b824c..34a21ff95472 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -179,9 +179,11 @@ struct mlx5_eswitch_fdb { struct mlx5_flow_namespace *ns; struct mlx5_flow_table *slow_fdb; struct mlx5_flow_group *send_to_vport_grp; + struct mlx5_flow_group *send_to_vport_meta_grp; struct mlx5_flow_group *peer_miss_grp; struct mlx5_flow_handle **peer_miss_rules; struct mlx5_flow_group *miss_grp; + struct mlx5_flow_handle **send_to_vport_meta_rules; struct mlx5_flow_handle *miss_rule_uni; struct mlx5_flow_handle *miss_rule_multi; int vlan_push_pop_refcount; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index a44728595420..94cb0217b4f3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -1080,6 +1080,81 @@ void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) mlx5_del_flow_rules(rule); } +static void mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw) +{ + struct mlx5_flow_handle **flows = esw->fdb_table.offloads.send_to_vport_meta_rules; + int i = 0, num_vfs = esw->esw_funcs.num_vfs, vport_num; + + if (!num_vfs || !flows) + return; + + mlx5_esw_for_each_vf_vport_num(esw, vport_num, num_vfs) + mlx5_del_flow_rules(flows[i++]); + + kvfree(flows); +} + +static int +mlx5_eswitch_add_send_to_vport_meta_rules(struct mlx5_eswitch *esw) +{ + int num_vfs, vport_num, rule_idx = 0, err = 0; + struct mlx5_flow_destination dest = {}; + struct mlx5_flow_act flow_act = {0}; + struct mlx5_flow_handle *flow_rule; + struct mlx5_flow_handle **flows; + struct mlx5_flow_spec *spec; + + num_vfs = esw->esw_funcs.num_vfs; + flows = kvzalloc(num_vfs * sizeof(*flows), GFP_KERNEL); + if (!flows) + return -ENOMEM; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { + err = -ENOMEM; + goto alloc_err; + } + + MLX5_SET(fte_match_param, spec->match_criteria, + misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); + MLX5_SET(fte_match_param, spec->match_criteria, + misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); + MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1, + ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK); + + spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; + dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + + mlx5_esw_for_each_vf_vport_num(esw, vport_num, num_vfs) { + MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0, + mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); + dest.vport.num = vport_num; + + flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, + spec, &flow_act, &dest, 1); + if (IS_ERR(flow_rule)) { + err = PTR_ERR(flow_rule); + esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule idx %d, err %ld\n", + rule_idx, PTR_ERR(flow_rule)); + goto rule_err; + } + flows[rule_idx++] = flow_rule; + } + + esw->fdb_table.offloads.send_to_vport_meta_rules = flows; + kvfree(spec); + return 0; + +rule_err: + while (--rule_idx >= 0) + mlx5_del_flow_rules(flows[rule_idx]); + kvfree(spec); +alloc_err: + kvfree(flows); + return err; +} + static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) { return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & @@ -1562,11 +1637,11 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_table_attr ft_attr = {}; + int num_vfs, table_size, ix, err = 0; struct mlx5_core_dev *dev = esw->dev; struct mlx5_flow_namespace *root_ns; struct mlx5_flow_table *fdb = NULL; u32 flags = 0, *flow_group_in; - int table_size, ix, err = 0; struct mlx5_flow_group *g; void *match_criteria; u8 *dmac; @@ -1592,7 +1667,7 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) } table_size = esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ + - MLX5_ESW_MISS_FLOWS + esw->total_vports; + MLX5_ESW_MISS_FLOWS + esw->total_vports + esw->esw_funcs.num_vfs; /* create the slow path fdb with encap set, so further table instances * can be created at run time while VFs are probed if the FW allows that. @@ -1640,6 +1715,38 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) } esw->fdb_table.offloads.send_to_vport_grp = g; + /* meta send to vport */ + memset(flow_group_in, 0, inlen); + MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, + MLX5_MATCH_MISC_PARAMETERS_2); + + match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); + + MLX5_SET(fte_match_param, match_criteria, + misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); + MLX5_SET(fte_match_param, match_criteria, + misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); + + num_vfs = esw->esw_funcs.num_vfs; + if (num_vfs) { + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix + num_vfs - 1); + ix += num_vfs; + + g = mlx5_create_flow_group(fdb, flow_group_in); + if (IS_ERR(g)) { + err = PTR_ERR(g); + esw_warn(dev, "Failed to create send-to-vport meta flow group err(%d)\n", + err); + goto send_vport_meta_err; + } + esw->fdb_table.offloads.send_to_vport_meta_grp = g; + + err = mlx5_eswitch_add_send_to_vport_meta_rules(esw); + if (err) + goto meta_rule_err; + } + if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { /* create peer esw miss group */ memset(flow_group_in, 0, inlen); @@ -1707,6 +1814,11 @@ miss_err: if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); peer_miss_err: + mlx5_eswitch_del_send_to_vport_meta_rules(esw); +meta_rule_err: + if (esw->fdb_table.offloads.send_to_vport_meta_grp) + mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); +send_vport_meta_err: mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); send_vport_err: esw_chains_destroy(esw, esw_chains(esw)); @@ -1728,7 +1840,10 @@ static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); + mlx5_eswitch_del_send_to_vport_meta_rules(esw); mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); + if (esw->fdb_table.offloads.send_to_vport_meta_grp) + mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h index 3b20e84049c1..994c2c8cb4fd 100644 --- a/include/linux/mlx5/eswitch.h +++ b/include/linux/mlx5/eswitch.h @@ -114,8 +114,16 @@ u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, #define ESW_ZONE_ID_BITS 8 #define ESW_TUN_OPTS_BITS 12 #define ESW_TUN_ID_BITS 12 -#define ESW_TUN_OFFSET ESW_ZONE_ID_BITS +#define ESW_TUN_OPTS_OFFSET ESW_ZONE_ID_BITS +#define ESW_TUN_OFFSET ESW_TUN_OPTS_OFFSET #define ESW_ZONE_ID_MASK GENMASK(ESW_ZONE_ID_BITS - 1, 0) +#define ESW_TUN_OPTS_MASK GENMASK(32 - ESW_TUN_ID_BITS - 1, ESW_TUN_OPTS_OFFSET) +#define ESW_TUN_MASK GENMASK(31, ESW_TUN_OFFSET) +#define ESW_TUN_ID_SLOW_TABLE_GOTO_VPORT 0 /* 0 is not a valid tunnel id */ +#define ESW_TUN_OPTS_SLOW_TABLE_GOTO_VPORT 0xFFF /* 0xFFF is a reserved mapping */ +#define ESW_TUN_SLOW_TABLE_GOTO_VPORT ((ESW_TUN_ID_SLOW_TABLE_GOTO_VPORT << ESW_TUN_OPTS_BITS) | \ + ESW_TUN_OPTS_SLOW_TABLE_GOTO_VPORT) +#define ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK ESW_TUN_OPTS_MASK u8 mlx5_eswitch_mode(struct mlx5_core_dev *dev); #else /* CONFIG_MLX5_ESWITCH */ -- cgit v1.2.3 From 52c2d15703c3a900d5f78cd599b823db40d5100b Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Tue, 19 Jan 2021 17:36:14 -0800 Subject: usb: common: Parse for USB SSP genXxY The USB "maximum-speed" property can now take the SSP signaling rate generation and lane count with these new strings: "super-speed-plus-gen2x2" "super-speed-plus-gen2x1" "super-speed-plus-gen1x2" Introduce usb_get_maximum_ssp_rate() to parse for the corresponding usb_ssp_rate enum. The original usb_get_maximum_speed() will return USB_SPEED_SUPER_PLUS if it matches one of these new strings. Signed-off-by: Thinh Nguyen Link: https://lore.kernel.org/r/f8ed896313d8cd8e2d2b540fc82db92b3ddf8a47.1611106162.git.Thinh.Nguyen@synopsys.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/common/common.c | 26 +++++++++++++++++++++++++- include/linux/usb/ch9.h | 11 +++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/usb/common/common.c b/drivers/usb/common/common.c index 1433260d99b4..fc21cf2d36f6 100644 --- a/drivers/usb/common/common.c +++ b/drivers/usb/common/common.c @@ -69,6 +69,13 @@ static const char *const speed_names[] = { [USB_SPEED_SUPER_PLUS] = "super-speed-plus", }; +static const char *const ssp_rate[] = { + [USB_SSP_GEN_UNKNOWN] = "UNKNOWN", + [USB_SSP_GEN_2x1] = "super-speed-plus-gen2x1", + [USB_SSP_GEN_1x2] = "super-speed-plus-gen1x2", + [USB_SSP_GEN_2x2] = "super-speed-plus-gen2x2", +}; + const char *usb_speed_string(enum usb_device_speed speed) { if (speed < 0 || speed >= ARRAY_SIZE(speed_names)) @@ -86,12 +93,29 @@ enum usb_device_speed usb_get_maximum_speed(struct device *dev) if (ret < 0) return USB_SPEED_UNKNOWN; - ret = match_string(speed_names, ARRAY_SIZE(speed_names), maximum_speed); + ret = match_string(ssp_rate, ARRAY_SIZE(ssp_rate), maximum_speed); + if (ret > 0) + return USB_SPEED_SUPER_PLUS; + ret = match_string(speed_names, ARRAY_SIZE(speed_names), maximum_speed); return (ret < 0) ? USB_SPEED_UNKNOWN : ret; } EXPORT_SYMBOL_GPL(usb_get_maximum_speed); +enum usb_ssp_rate usb_get_maximum_ssp_rate(struct device *dev) +{ + const char *maximum_speed; + int ret; + + ret = device_property_read_string(dev, "maximum-speed", &maximum_speed); + if (ret < 0) + return USB_SSP_GEN_UNKNOWN; + + ret = match_string(ssp_rate, ARRAY_SIZE(ssp_rate), maximum_speed); + return (ret < 0) ? USB_SSP_GEN_UNKNOWN : ret; +} +EXPORT_SYMBOL_GPL(usb_get_maximum_ssp_rate); + const char *usb_state_string(enum usb_device_state state) { static const char *const names[] = { diff --git a/include/linux/usb/ch9.h b/include/linux/usb/ch9.h index 86c50907634e..abdd310c77f0 100644 --- a/include/linux/usb/ch9.h +++ b/include/linux/usb/ch9.h @@ -71,6 +71,17 @@ extern const char *usb_speed_string(enum usb_device_speed speed); */ extern enum usb_device_speed usb_get_maximum_speed(struct device *dev); +/** + * usb_get_maximum_ssp_rate - Get the signaling rate generation and lane count + * of a SuperSpeed Plus capable device. + * @dev: Pointer to the given USB controller device + * + * If the string from "maximum-speed" property is super-speed-plus-genXxY where + * 'X' is the generation number and 'Y' is the number of lanes, then this + * function returns the corresponding enum usb_ssp_rate. + */ +extern enum usb_ssp_rate usb_get_maximum_ssp_rate(struct device *dev); + /** * usb_state_string - Returns human readable name for the state. * @state: The state to return a human-readable name for. If it's not -- cgit v1.2.3 From b358e2122b9d7aa99f681d4edfafd999845d16ff Mon Sep 17 00:00:00 2001 From: Kevin Hao Date: Thu, 4 Feb 2021 18:56:35 +0800 Subject: mm: page_frag: Introduce page_frag_alloc_align() In the current implementation of page_frag_alloc(), it doesn't have any align guarantee for the returned buffer address. But for some hardwares they do require the DMA buffer to be aligned correctly, so we would have to use some workarounds like below if the buffers allocated by the page_frag_alloc() are used by these hardwares for DMA. buf = page_frag_alloc(really_needed_size + align); buf = PTR_ALIGN(buf, align); These codes seems ugly and would waste a lot of memories if the buffers are used in a network driver for the TX/RX. So introduce page_frag_alloc_align() to make sure that an aligned buffer address is returned. Signed-off-by: Kevin Hao Acked-by: Vlastimil Babka Reviewed-by: Alexander Duyck Signed-off-by: Jakub Kicinski --- include/linux/gfp.h | 12 ++++++++++-- mm/page_alloc.c | 8 +++++--- 2 files changed, 15 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 6e479e9c48ce..80544d5c08e7 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -583,8 +583,16 @@ extern void free_pages(unsigned long addr, unsigned int order); struct page_frag_cache; extern void __page_frag_cache_drain(struct page *page, unsigned int count); -extern void *page_frag_alloc(struct page_frag_cache *nc, - unsigned int fragsz, gfp_t gfp_mask); +extern void *page_frag_alloc_align(struct page_frag_cache *nc, + unsigned int fragsz, gfp_t gfp_mask, + unsigned int align_mask); + +static inline void *page_frag_alloc(struct page_frag_cache *nc, + unsigned int fragsz, gfp_t gfp_mask) +{ + return page_frag_alloc_align(nc, fragsz, gfp_mask, ~0u); +} + extern void page_frag_free(void *addr); #define __free_page(page) __free_pages((page), 0) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 519a60d5b6f7..ef5070fed76b 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5137,8 +5137,9 @@ void __page_frag_cache_drain(struct page *page, unsigned int count) } EXPORT_SYMBOL(__page_frag_cache_drain); -void *page_frag_alloc(struct page_frag_cache *nc, - unsigned int fragsz, gfp_t gfp_mask) +void *page_frag_alloc_align(struct page_frag_cache *nc, + unsigned int fragsz, gfp_t gfp_mask, + unsigned int align_mask) { unsigned int size = PAGE_SIZE; struct page *page; @@ -5190,11 +5191,12 @@ refill: } nc->pagecnt_bias--; + offset &= align_mask; nc->offset = offset; return nc->va + offset; } -EXPORT_SYMBOL(page_frag_alloc); +EXPORT_SYMBOL(page_frag_alloc_align); /* * Frees a page fragment allocated out of either a compound or order 0 page. -- cgit v1.2.3 From 3f6e687dff395da43b056c18150a423bc7bf5d14 Mon Sep 17 00:00:00 2001 From: Kevin Hao Date: Thu, 4 Feb 2021 18:56:36 +0800 Subject: net: Introduce {netdev,napi}_alloc_frag_align() In the current implementation of {netdev,napi}_alloc_frag(), it doesn't have any align guarantee for the returned buffer address, But for some hardwares they do require the DMA buffer to be aligned correctly, so we would have to use some workarounds like below if the buffers allocated by the {netdev,napi}_alloc_frag() are used by these hardwares for DMA. buf = napi_alloc_frag(really_needed_size + align); buf = PTR_ALIGN(buf, align); These codes seems ugly and would waste a lot of memories if the buffers are used in a network driver for the TX/RX. We have added the align support for the page_frag functions, so add the corresponding {netdev,napi}_frag functions. Signed-off-by: Kevin Hao Reviewed-by: Alexander Duyck Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 36 ++++++++++++++++++++++++++++++++++-- net/core/skbuff.c | 26 ++++++++++---------------- 2 files changed, 44 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 0e42c53b8ca9..0a4e91a2f873 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2818,7 +2818,26 @@ void skb_queue_purge(struct sk_buff_head *list); unsigned int skb_rbtree_purge(struct rb_root *root); -void *netdev_alloc_frag(unsigned int fragsz); +void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask); + +/** + * netdev_alloc_frag - allocate a page fragment + * @fragsz: fragment size + * + * Allocates a frag from a page for receive buffer. + * Uses GFP_ATOMIC allocations. + */ +static inline void *netdev_alloc_frag(unsigned int fragsz) +{ + return __netdev_alloc_frag_align(fragsz, ~0u); +} + +static inline void *netdev_alloc_frag_align(unsigned int fragsz, + unsigned int align) +{ + WARN_ON_ONCE(!is_power_of_2(align)); + return __netdev_alloc_frag_align(fragsz, -align); +} struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length, gfp_t gfp_mask); @@ -2877,7 +2896,20 @@ static inline void skb_free_frag(void *addr) page_frag_free(addr); } -void *napi_alloc_frag(unsigned int fragsz); +void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask); + +static inline void *napi_alloc_frag(unsigned int fragsz) +{ + return __napi_alloc_frag_align(fragsz, ~0u); +} + +static inline void *napi_alloc_frag_align(unsigned int fragsz, + unsigned int align) +{ + WARN_ON_ONCE(!is_power_of_2(align)); + return __napi_alloc_frag_align(fragsz, -align); +} + struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int length, gfp_t gfp_mask); static inline struct sk_buff *napi_alloc_skb(struct napi_struct *napi, diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 3787093239f5..d380c7b5a12d 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -374,29 +374,23 @@ struct napi_alloc_cache { static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); -static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask) +static void *__alloc_frag_align(unsigned int fragsz, gfp_t gfp_mask, + unsigned int align_mask) { struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); - return page_frag_alloc(&nc->page, fragsz, gfp_mask); + return page_frag_alloc_align(&nc->page, fragsz, gfp_mask, align_mask); } -void *napi_alloc_frag(unsigned int fragsz) +void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) { fragsz = SKB_DATA_ALIGN(fragsz); - return __napi_alloc_frag(fragsz, GFP_ATOMIC); + return __alloc_frag_align(fragsz, GFP_ATOMIC, align_mask); } -EXPORT_SYMBOL(napi_alloc_frag); +EXPORT_SYMBOL(__napi_alloc_frag_align); -/** - * netdev_alloc_frag - allocate a page fragment - * @fragsz: fragment size - * - * Allocates a frag from a page for receive buffer. - * Uses GFP_ATOMIC allocations. - */ -void *netdev_alloc_frag(unsigned int fragsz) +void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) { struct page_frag_cache *nc; void *data; @@ -404,15 +398,15 @@ void *netdev_alloc_frag(unsigned int fragsz) fragsz = SKB_DATA_ALIGN(fragsz); if (in_irq() || irqs_disabled()) { nc = this_cpu_ptr(&netdev_alloc_cache); - data = page_frag_alloc(nc, fragsz, GFP_ATOMIC); + data = page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, align_mask); } else { local_bh_disable(); - data = __napi_alloc_frag(fragsz, GFP_ATOMIC); + data = __alloc_frag_align(fragsz, GFP_ATOMIC, align_mask); local_bh_enable(); } return data; } -EXPORT_SYMBOL(netdev_alloc_frag); +EXPORT_SYMBOL(__netdev_alloc_frag_align); /** * __netdev_alloc_skb - allocate an skbuff for rx on a specific device -- cgit v1.2.3 From e17fe6579de023725ec22a16965e9099e4a05ac9 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 15 Jan 2021 10:18:16 -0800 Subject: fs-verity: add FS_IOC_READ_VERITY_METADATA ioctl Add an ioctl FS_IOC_READ_VERITY_METADATA which will allow reading verity metadata from a file that has fs-verity enabled, including: - The Merkle tree - The fsverity_descriptor (not including the signature if present) - The built-in signature, if present This ioctl has similar semantics to pread(). It is passed the type of metadata to read (one of the above three), and a buffer, offset, and size. It returns the number of bytes read or an error. Separate patches will add support for each of the above metadata types. This patch just adds the ioctl itself. This ioctl doesn't make any assumption about where the metadata is stored on-disk. It does assume the metadata is in a stable format, but that's basically already the case: - The Merkle tree and fsverity_descriptor are defined by how fs-verity file digests are computed; see the "File digest computation" section of Documentation/filesystems/fsverity.rst. Technically, the way in which the levels of the tree are ordered relative to each other wasn't previously specified, but it's logical to put the root level first. - The built-in signature is the value passed to FS_IOC_ENABLE_VERITY. This ioctl is useful because it allows writing a server program that takes a verity file and serves it to a client program, such that the client can do its own fs-verity compatible verification of the file. This only makes sense if the client doesn't trust the server and if the server needs to provide the storage for the client. More concretely, there is interest in using this ability in Android to export APK files (which are protected by fs-verity) to "protected VMs". This would use Protected KVM (https://lwn.net/Articles/836693), which provides an isolated execution environment without having to trust the traditional "host". A "guest" VM can boot from a signed image and perform specific tasks in a minimum trusted environment using files that have fs-verity enabled on the host, without trusting the host or requiring that the guest has its own trusted storage. Technically, it would be possible to duplicate the metadata and store it in separate files for serving. However, that would be less efficient and would require extra care in userspace to maintain file consistency. In addition to the above, the ability to read the built-in signatures is useful because it allows a system that is using the in-kernel signature verification to migrate to userspace signature verification. Link: https://lore.kernel.org/r/20210115181819.34732-4-ebiggers@kernel.org Reviewed-by: Victor Hsieh Acked-by: Jaegeuk Kim Reviewed-by: Chao Yu Signed-off-by: Eric Biggers --- Documentation/filesystems/fsverity.rst | 57 ++++++++++++++++++++++++++++++++++ fs/ext4/ioctl.c | 7 +++++ fs/f2fs/file.c | 11 +++++++ fs/verity/Makefile | 1 + fs/verity/read_metadata.c | 55 ++++++++++++++++++++++++++++++++ include/linux/fsverity.h | 12 +++++++ include/uapi/linux/fsverity.h | 10 ++++++ 7 files changed, 153 insertions(+) create mode 100644 fs/verity/read_metadata.c (limited to 'include/linux') diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index e0204a23e997..9ef7a7de6008 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -217,6 +217,63 @@ FS_IOC_MEASURE_VERITY can fail with the following errors: - ``EOVERFLOW``: the digest is longer than the specified ``digest_size`` bytes. Try providing a larger buffer. +FS_IOC_READ_VERITY_METADATA +--------------------------- + +The FS_IOC_READ_VERITY_METADATA ioctl reads verity metadata from a +verity file. This ioctl is available since Linux v5.12. + +This ioctl allows writing a server program that takes a verity file +and serves it to a client program, such that the client can do its own +fs-verity compatible verification of the file. This only makes sense +if the client doesn't trust the server and if the server needs to +provide the storage for the client. + +This is a fairly specialized use case, and most fs-verity users won't +need this ioctl. + +This ioctl takes in a pointer to the following structure:: + + struct fsverity_read_metadata_arg { + __u64 metadata_type; + __u64 offset; + __u64 length; + __u64 buf_ptr; + __u64 __reserved; + }; + +``metadata_type`` specifies the type of metadata to read. + +The semantics are similar to those of ``pread()``. ``offset`` +specifies the offset in bytes into the metadata item to read from, and +``length`` specifies the maximum number of bytes to read from the +metadata item. ``buf_ptr`` is the pointer to the buffer to read into, +cast to a 64-bit integer. ``__reserved`` must be 0. On success, the +number of bytes read is returned. 0 is returned at the end of the +metadata item. The returned length may be less than ``length``, for +example if the ioctl is interrupted. + +The metadata returned by FS_IOC_READ_VERITY_METADATA isn't guaranteed +to be authenticated against the file digest that would be returned by +`FS_IOC_MEASURE_VERITY`_, as the metadata is expected to be used to +implement fs-verity compatible verification anyway (though absent a +malicious disk, the metadata will indeed match). E.g. to implement +this ioctl, the filesystem is allowed to just read the Merkle tree +blocks from disk without actually verifying the path to the root node. + +FS_IOC_READ_VERITY_METADATA can fail with the following errors: + +- ``EFAULT``: the caller provided inaccessible memory +- ``EINTR``: the ioctl was interrupted before any data was read +- ``EINVAL``: reserved fields were set, or ``offset + length`` + overflowed +- ``ENODATA``: the file is not a verity file +- ``ENOTTY``: this type of filesystem does not implement fs-verity, or + this ioctl is not yet implemented on it +- ``EOPNOTSUPP``: the kernel was not configured with fs-verity + support, or the filesystem superblock has not had the 'verity' + feature enabled on it. (See `Filesystem support`_.) + FS_IOC_GETFLAGS --------------- diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index d9665d2f82db..713b1ae44c1a 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -1309,6 +1309,12 @@ out: return -EOPNOTSUPP; return fsverity_ioctl_measure(filp, (void __user *)arg); + case FS_IOC_READ_VERITY_METADATA: + if (!ext4_has_feature_verity(sb)) + return -EOPNOTSUPP; + return fsverity_ioctl_read_metadata(filp, + (const void __user *)arg); + default: return -ENOTTY; } @@ -1391,6 +1397,7 @@ long ext4_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case FS_IOC_GETFSMAP: case FS_IOC_ENABLE_VERITY: case FS_IOC_MEASURE_VERITY: + case FS_IOC_READ_VERITY_METADATA: case EXT4_IOC_CLEAR_ES_CACHE: case EXT4_IOC_GETSTATE: case EXT4_IOC_GET_ES_CACHE: diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index f585545277d7..d0aefb5b97fa 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -3357,6 +3357,14 @@ static int f2fs_ioc_measure_verity(struct file *filp, unsigned long arg) return fsverity_ioctl_measure(filp, (void __user *)arg); } +static int f2fs_ioc_read_verity_metadata(struct file *filp, unsigned long arg) +{ + if (!f2fs_sb_has_verity(F2FS_I_SB(file_inode(filp)))) + return -EOPNOTSUPP; + + return fsverity_ioctl_read_metadata(filp, (const void __user *)arg); +} + static int f2fs_ioc_getfslabel(struct file *filp, unsigned long arg) { struct inode *inode = file_inode(filp); @@ -4272,6 +4280,8 @@ static long __f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) return f2fs_ioc_enable_verity(filp, arg); case FS_IOC_MEASURE_VERITY: return f2fs_ioc_measure_verity(filp, arg); + case FS_IOC_READ_VERITY_METADATA: + return f2fs_ioc_read_verity_metadata(filp, arg); case FS_IOC_GETFSLABEL: return f2fs_ioc_getfslabel(filp, arg); case FS_IOC_SETFSLABEL: @@ -4523,6 +4533,7 @@ long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case F2FS_IOC_RESIZE_FS: case FS_IOC_ENABLE_VERITY: case FS_IOC_MEASURE_VERITY: + case FS_IOC_READ_VERITY_METADATA: case FS_IOC_GETFSLABEL: case FS_IOC_SETFSLABEL: case F2FS_IOC_GET_COMPRESS_BLOCKS: diff --git a/fs/verity/Makefile b/fs/verity/Makefile index 570e9136334d..435559a4fa9e 100644 --- a/fs/verity/Makefile +++ b/fs/verity/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_FS_VERITY) += enable.o \ init.o \ measure.o \ open.o \ + read_metadata.o \ verify.o obj-$(CONFIG_FS_VERITY_BUILTIN_SIGNATURES) += signature.o diff --git a/fs/verity/read_metadata.c b/fs/verity/read_metadata.c new file mode 100644 index 000000000000..43be990fd53e --- /dev/null +++ b/fs/verity/read_metadata.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Ioctl to read verity metadata + * + * Copyright 2021 Google LLC + */ + +#include "fsverity_private.h" + +#include + +/** + * fsverity_ioctl_read_metadata() - read verity metadata from a file + * @filp: file to read the metadata from + * @uarg: user pointer to fsverity_read_metadata_arg + * + * Return: length read on success, 0 on EOF, -errno on failure + */ +int fsverity_ioctl_read_metadata(struct file *filp, const void __user *uarg) +{ + struct inode *inode = file_inode(filp); + const struct fsverity_info *vi; + struct fsverity_read_metadata_arg arg; + int length; + void __user *buf; + + vi = fsverity_get_info(inode); + if (!vi) + return -ENODATA; /* not a verity file */ + /* + * Note that we don't have to explicitly check that the file is open for + * reading, since verity files can only be opened for reading. + */ + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + if (arg.__reserved) + return -EINVAL; + + /* offset + length must not overflow. */ + if (arg.offset + arg.length < arg.offset) + return -EINVAL; + + /* Ensure that the return value will fit in INT_MAX. */ + length = min_t(u64, arg.length, INT_MAX); + + buf = u64_to_user_ptr(arg.buf_ptr); + + switch (arg.metadata_type) { + default: + return -EINVAL; + } +} +EXPORT_SYMBOL_GPL(fsverity_ioctl_read_metadata); diff --git a/include/linux/fsverity.h b/include/linux/fsverity.h index c1144a450392..b568b3c7d095 100644 --- a/include/linux/fsverity.h +++ b/include/linux/fsverity.h @@ -138,6 +138,10 @@ int fsverity_file_open(struct inode *inode, struct file *filp); int fsverity_prepare_setattr(struct dentry *dentry, struct iattr *attr); void fsverity_cleanup_inode(struct inode *inode); +/* read_metadata.c */ + +int fsverity_ioctl_read_metadata(struct file *filp, const void __user *uarg); + /* verify.c */ bool fsverity_verify_page(struct page *page); @@ -183,6 +187,14 @@ static inline void fsverity_cleanup_inode(struct inode *inode) { } +/* read_metadata.c */ + +static inline int fsverity_ioctl_read_metadata(struct file *filp, + const void __user *uarg) +{ + return -EOPNOTSUPP; +} + /* verify.c */ static inline bool fsverity_verify_page(struct page *page) diff --git a/include/uapi/linux/fsverity.h b/include/uapi/linux/fsverity.h index 33f44156f8ea..e062751294d0 100644 --- a/include/uapi/linux/fsverity.h +++ b/include/uapi/linux/fsverity.h @@ -83,7 +83,17 @@ struct fsverity_formatted_digest { __u8 digest[]; }; +struct fsverity_read_metadata_arg { + __u64 metadata_type; + __u64 offset; + __u64 length; + __u64 buf_ptr; + __u64 __reserved; +}; + #define FS_IOC_ENABLE_VERITY _IOW('f', 133, struct fsverity_enable_arg) #define FS_IOC_MEASURE_VERITY _IOWR('f', 134, struct fsverity_digest) +#define FS_IOC_READ_VERITY_METADATA \ + _IOWR('f', 135, struct fsverity_read_metadata_arg) #endif /* _UAPI_LINUX_FSVERITY_H */ -- cgit v1.2.3 From 4331667fa14e6643859d0498b34281185eb8018b Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Fri, 5 Feb 2021 14:56:39 +0800 Subject: ssb: Use true and false for bool variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the following coccicheck warnings: ./include/linux/ssb/ssb_driver_gige.h:89:8-9: WARNING: return of 0/1 in function 'ssb_gige_one_dma_at_once' with return type bool. ./include/linux/ssb/ssb_driver_gige.h:79:8-9: WARNING: return of 0/1 in function 'ssb_gige_have_roboswitch' with return type bool. ./include/linux/ssb/ssb_driver_gige.h:182:8-9: WARNING: return of 0/1 in function 'ssb_gige_must_flush_posted_writes' with return type bool. ./include/linux/ssb/ssb_driver_gige.h:178:8-9: WARNING: return of 0/1 in function 'ssb_gige_one_dma_at_once' with return type bool. ./include/linux/ssb/ssb_driver_gige.h:174:8-9: WARNING: return of 0/1 in function 'ssb_gige_have_roboswitch' with return type bool. ./include/linux/ssb/ssb_driver_gige.h:170:8-9: WARNING: return of 0/1 in function 'ssb_gige_is_rgmii' with return type bool. ./include/linux/ssb/ssb_driver_gige.h:162:8-9: WARNING: return of 0/1 in function 'pdev_is_ssb_gige_core' with return type bool. Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Acked-by: Michael Büsch Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/1612508199-92282-1-git-send-email-jiapeng.chong@linux.alibaba.com --- include/linux/ssb/ssb_driver_gige.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ssb/ssb_driver_gige.h b/include/linux/ssb/ssb_driver_gige.h index 31593b34608e..15ba0df1ee0d 100644 --- a/include/linux/ssb/ssb_driver_gige.h +++ b/include/linux/ssb/ssb_driver_gige.h @@ -76,7 +76,7 @@ static inline bool ssb_gige_have_roboswitch(struct pci_dev *pdev) if (dev) return !!(dev->dev->bus->sprom.boardflags_lo & SSB_GIGE_BFL_ROBOSWITCH); - return 0; + return false; } /* Returns whether we can only do one DMA at once. */ @@ -86,7 +86,7 @@ static inline bool ssb_gige_one_dma_at_once(struct pci_dev *pdev) if (dev) return ((dev->dev->bus->chip_id == 0x4785) && (dev->dev->bus->chip_rev < 2)); - return 0; + return false; } /* Returns whether we must flush posted writes. */ @@ -159,7 +159,7 @@ static inline void ssb_gige_exit(void) static inline bool pdev_is_ssb_gige_core(struct pci_dev *pdev) { - return 0; + return false; } static inline struct ssb_gige * pdev_to_ssb_gige(struct pci_dev *pdev) { @@ -167,19 +167,19 @@ static inline struct ssb_gige * pdev_to_ssb_gige(struct pci_dev *pdev) } static inline bool ssb_gige_is_rgmii(struct pci_dev *pdev) { - return 0; + return false; } static inline bool ssb_gige_have_roboswitch(struct pci_dev *pdev) { - return 0; + return false; } static inline bool ssb_gige_one_dma_at_once(struct pci_dev *pdev) { - return 0; + return false; } static inline bool ssb_gige_must_flush_posted_writes(struct pci_dev *pdev) { - return 0; + return false; } static inline int ssb_gige_get_macaddr(struct pci_dev *pdev, u8 *macaddr) { -- cgit v1.2.3 From 6ac0b71537e1c14e7532408fe4aae553aa314237 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 2 Feb 2021 18:19:20 +0100 Subject: block: move struct biovec_slab to bio.c struct biovec_slab is only used inside of bio.c, so move it there. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/bio.c | 6 ++++++ include/linux/bio.h | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/block/bio.c b/block/bio.c index cee2d310f02e..2c359dadfdf6 100644 --- a/block/bio.c +++ b/block/bio.c @@ -25,6 +25,12 @@ #include "blk.h" #include "blk-rq-qos.h" +struct biovec_slab { + int nr_vecs; + char *name; + struct kmem_cache *slab; +}; + /* * if you change this list, also change bvec_alloc or things will * break badly! cannot be bigger than what you can fit into an diff --git a/include/linux/bio.h b/include/linux/bio.h index c74857cf1252..4a84207dd996 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -720,12 +720,6 @@ struct bio_set { struct workqueue_struct *rescue_workqueue; }; -struct biovec_slab { - int nr_vecs; - char *name; - struct kmem_cache *slab; -}; - static inline bool bioset_initialized(struct bio_set *bs) { return bs->bio_slab != NULL; -- cgit v1.2.3 From 0f2e6ab851ae146c468bc5151c302c6e2473f70a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 2 Feb 2021 18:19:24 +0100 Subject: block: turn the nr_iovecs argument to bio_alloc* into an unsigned short The bi_max_vecs and bi_vcnt fields are defined as unsigned short, so don't allow passing larger values in. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/bio.c | 4 ++-- include/linux/bio.h | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/block/bio.c b/block/bio.c index ae241252ea14..3d28d4723f6f 100644 --- a/block/bio.c +++ b/block/bio.c @@ -407,7 +407,7 @@ static void punt_bios_to_rescuer(struct bio_set *bs) * * Returns: Pointer to new bio on success, NULL on failure. */ -struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs, +struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned short nr_iovecs, struct bio_set *bs) { gfp_t saved_gfp = gfp_mask; @@ -493,7 +493,7 @@ EXPORT_SYMBOL(bio_alloc_bioset); * * Returns: Pointer to new bio on success, NULL on failure. */ -struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs) +struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs) { struct bio *bio; diff --git a/include/linux/bio.h b/include/linux/bio.h index 4a84207dd996..9ceeb8ecdb7f 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -407,8 +407,9 @@ extern void bioset_exit(struct bio_set *); extern int biovec_init_pool(mempool_t *pool, int pool_entries); extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src); -extern struct bio *bio_alloc_bioset(gfp_t, unsigned int, struct bio_set *); -struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs); +struct bio *bio_alloc_bioset(gfp_t gfp, unsigned short nr_iovecs, + struct bio_set *bs); +struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs); extern void bio_put(struct bio *); extern void __bio_clone_fast(struct bio *, struct bio *); @@ -416,7 +417,7 @@ extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *); extern struct bio_set fs_bio_set; -static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs) +static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned short nr_iovecs) { return bio_alloc_bioset(gfp_mask, nr_iovecs, &fs_bio_set); } -- cgit v1.2.3 From 7a800a20ae6329e803c5c646b20811a6ae9ca136 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 2 Feb 2021 18:19:29 +0100 Subject: block: use bi_max_vecs to find the bvec pool Instead of encoding of the bvec pool using magic bio flags, just use a helper to find the pool based on the max_vecs value. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/bio-integrity.c | 11 ++--- block/bio.c | 104 +++++++++++++++++++--------------------------- block/blk.h | 6 +-- include/linux/bio.h | 1 - include/linux/blk_types.h | 29 +------------ 5 files changed, 51 insertions(+), 100 deletions(-) (limited to 'include/linux') diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 19617fa326c3..dfa652122a2d 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -28,7 +28,7 @@ static void __bio_integrity_free(struct bio_set *bs, if (bs && mempool_initialized(&bs->bio_integrity_pool)) { if (bip->bip_vec) bvec_free(&bs->bvec_integrity_pool, bip->bip_vec, - bip->bip_slab); + bip->bip_max_vcnt); mempool_free(bip, &bs->bio_integrity_pool); } else { kfree(bip); @@ -70,14 +70,11 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, memset(bip, 0, sizeof(*bip)); if (nr_vecs > inline_vecs) { - unsigned long idx = 0; - - bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx, - &bs->bvec_integrity_pool); + bip->bip_max_vcnt = nr_vecs; + bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool, + &bip->bip_max_vcnt, gfp_mask); if (!bip->bip_vec) goto err; - bip->bip_max_vcnt = bvec_nr_vecs(idx); - bip->bip_slab = idx; } else { bip->bip_vec = bip->bip_inline_vecs; bip->bip_max_vcnt = inline_vecs; diff --git a/block/bio.c b/block/bio.c index a36f955cd120..a0eabe2f8b07 100644 --- a/block/bio.c +++ b/block/bio.c @@ -36,6 +36,24 @@ static struct biovec_slab { { .nr_vecs = BIO_MAX_PAGES, .name = "biovec-max" }, }; +static struct biovec_slab *biovec_slab(unsigned short nr_vecs) +{ + switch (nr_vecs) { + /* smaller bios use inline vecs */ + case 5 ... 16: + return &bvec_slabs[0]; + case 17 ... 64: + return &bvec_slabs[1]; + case 65 ... 128: + return &bvec_slabs[2]; + case 129 ... BIO_MAX_PAGES: + return &bvec_slabs[3]; + default: + BUG(); + return NULL; + } +} + /* * fs_bio_set is the bio_set containing bio and iovec memory pools used by * IO code that does not need private memory pools. @@ -131,26 +149,14 @@ out: mutex_unlock(&bio_slab_lock); } -unsigned int bvec_nr_vecs(unsigned short idx) +void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs) { - return bvec_slabs[--idx].nr_vecs; -} + BIO_BUG_ON(nr_vecs > BIO_MAX_PAGES); -void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx) -{ - if (!idx) - return; - idx--; - - BIO_BUG_ON(idx >= BVEC_POOL_NR); - - if (idx == BVEC_POOL_MAX) { + if (nr_vecs == BIO_MAX_PAGES) mempool_free(bv, pool); - } else { - struct biovec_slab *bvs = bvec_slabs + idx; - - kmem_cache_free(bvs->slab, bv); - } + else if (nr_vecs > BIO_INLINE_VECS) + kmem_cache_free(biovec_slab(nr_vecs)->slab, bv); } /* @@ -163,48 +169,34 @@ static inline gfp_t bvec_alloc_gfp(gfp_t gfp) __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; } -struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx, - mempool_t *pool) +struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, + gfp_t gfp_mask) { + struct biovec_slab *bvs = biovec_slab(*nr_vecs); + + if (WARN_ON_ONCE(!bvs)) + return NULL; + /* - * see comment near bvec_array define! + * Upgrade the nr_vecs request to take full advantage of the allocation. + * We also rely on this in the bvec_free path. */ - switch (nr) { - /* smaller bios use inline vecs */ - case 5 ... 16: - *idx = 2; - break; - case 17 ... 64: - *idx = 3; - break; - case 65 ... 128: - *idx = 4; - break; - case 129 ... BIO_MAX_PAGES: - *idx = 5; - break; - default: - return NULL; - } + *nr_vecs = bvs->nr_vecs; /* * Try a slab allocation first for all smaller allocations. If that * fails and __GFP_DIRECT_RECLAIM is set retry with the mempool. * The mempool is sized to handle up to BIO_MAX_PAGES entries. */ - if (*idx < BVEC_POOL_MAX) { - struct biovec_slab *bvs = bvec_slabs + *idx; + if (*nr_vecs < BIO_MAX_PAGES) { struct bio_vec *bvl; bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask)); - if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM)) { - (*idx)++; + if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM)) return bvl; - } - *idx = BVEC_POOL_MAX; + *nr_vecs = BIO_MAX_PAGES; } - (*idx)++; return mempool_alloc(pool, gfp_mask); } @@ -231,7 +223,7 @@ static void bio_free(struct bio *bio) bio_uninit(bio); if (bs) { - bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio)); + bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs); /* * If we have front padding, adjust the bio pointer before freeing @@ -275,12 +267,8 @@ EXPORT_SYMBOL(bio_init); */ void bio_reset(struct bio *bio) { - unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS); - bio_uninit(bio); - memset(bio, 0, BIO_RESET_BYTES); - bio->bi_flags = flags; atomic_set(&bio->__bi_remaining, 1); } EXPORT_SYMBOL(bio_reset); @@ -453,22 +441,18 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned short nr_iovecs, bio = p + bs->front_pad; if (nr_iovecs > BIO_INLINE_VECS) { - unsigned long idx = 0; struct bio_vec *bvl = NULL; - bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool); + bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask); if (!bvl && gfp_mask != saved_gfp) { punt_bios_to_rescuer(bs); gfp_mask = saved_gfp; - bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, - &bs->bvec_pool); + bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask); } - if (unlikely(!bvl)) goto err_free; - bio_init(bio, bvl, bvec_nr_vecs(idx)); - bio->bi_flags |= idx << BVEC_POOL_OFFSET; + bio_init(bio, bvl, nr_iovecs); } else if (nr_iovecs) { bio_init(bio, bio->bi_inline_vecs, BIO_INLINE_VECS); } else { @@ -644,7 +628,7 @@ EXPORT_SYMBOL(bio_put); */ void __bio_clone_fast(struct bio *bio, struct bio *bio_src) { - BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio)); + WARN_ON_ONCE(bio->bi_pool && bio->bi_max_vecs); /* * most users will be overriding ->bi_bdev with a new target, @@ -934,7 +918,7 @@ EXPORT_SYMBOL_GPL(bio_release_pages); static int bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter) { - WARN_ON_ONCE(BVEC_POOL_IDX(bio) != 0); + WARN_ON_ONCE(bio->bi_max_vecs); bio->bi_vcnt = iter->nr_segs; bio->bi_io_vec = (struct bio_vec *)iter->bvec; @@ -1495,7 +1479,7 @@ EXPORT_SYMBOL_GPL(bio_trim); */ int biovec_init_pool(mempool_t *pool, int pool_entries) { - struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX; + struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1; return mempool_init_slab_pool(pool, pool_entries, bp->slab); } @@ -1605,8 +1589,6 @@ static int __init init_bio(void) { int i; - BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET); - bio_integrity_init(); for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) { diff --git a/block/blk.h b/block/blk.h index e022a0d0f2ce..bfc4d526f626 100644 --- a/block/blk.h +++ b/block/blk.h @@ -56,9 +56,9 @@ void blk_free_flush_queue(struct blk_flush_queue *q); void blk_freeze_queue(struct request_queue *q); #define BIO_INLINE_VECS 4 -struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *); -void bvec_free(mempool_t *, struct bio_vec *, unsigned int); -unsigned int bvec_nr_vecs(unsigned short idx); +struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, + gfp_t gfp_mask); +void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs); static inline bool biovec_phys_mergeable(struct request_queue *q, struct bio_vec *vec1, struct bio_vec *vec2) diff --git a/include/linux/bio.h b/include/linux/bio.h index 9ceeb8ecdb7f..3cbbaf76906e 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -329,7 +329,6 @@ struct bio_integrity_payload { struct bvec_iter bip_iter; - unsigned short bip_slab; /* slab the bip came from */ unsigned short bip_vcnt; /* # of integrity bio_vecs */ unsigned short bip_max_vcnt; /* integrity bio_vec slots */ unsigned short bip_flags; /* control flags */ diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 1bc6f6a01070..db026b6ec15a 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -227,7 +227,7 @@ struct bio { * top bits REQ_OP. Use * accessors. */ - unsigned short bi_flags; /* status, etc and bvec pool number */ + unsigned short bi_flags; /* BIO_* below */ unsigned short bi_ioprio; unsigned short bi_write_hint; blk_status_t bi_status; @@ -307,33 +307,6 @@ enum { BIO_FLAG_LAST }; -/* See BVEC_POOL_OFFSET below before adding new flags */ - -/* - * We support 6 different bvec pools, the last one is magic in that it - * is backed by a mempool. - */ -#define BVEC_POOL_NR 6 -#define BVEC_POOL_MAX (BVEC_POOL_NR - 1) - -/* - * Top 3 bits of bio flags indicate the pool the bvecs came from. We add - * 1 to the actual index so that 0 indicates that there are no bvecs to be - * freed. - */ -#define BVEC_POOL_BITS (3) -#define BVEC_POOL_OFFSET (16 - BVEC_POOL_BITS) -#define BVEC_POOL_IDX(bio) ((bio)->bi_flags >> BVEC_POOL_OFFSET) -#if (1<< BVEC_POOL_BITS) < (BVEC_POOL_NR+1) -# error "BVEC_POOL_BITS is too small" -#endif - -/* - * Flags starting here get preserved by bio_reset() - this includes - * only BVEC_POOL_IDX() - */ -#define BIO_RESET_BITS BVEC_POOL_OFFSET - typedef __u32 __bitwise blk_mq_req_flags_t; /* -- cgit v1.2.3 From 71c36788b9253f086d09763b98804ed473e12a3b Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 22 Jan 2021 11:58:04 +0200 Subject: lib/zstd: convert constants to defines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These constants are really used internally by zstd and including linux/zstd.h into users results in the following warnings: In file included from fs/btrfs/zstd.c:19: ./include/linux/zstd.h:798:21: warning: ‘ZSTD_skippableHeaderSize’ defined but not used [-Wunused-const-variable=] 798 | static const size_t ZSTD_skippableHeaderSize = 8; | ^~~~~~~~~~~~~~~~~~~~~~~~ ./include/linux/zstd.h:796:21: warning: ‘ZSTD_frameHeaderSize_max’ defined but not used [-Wunused-const-variable=] 796 | static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; | ^~~~~~~~~~~~~~~~~~~~~~~~ ./include/linux/zstd.h:795:21: warning: ‘ZSTD_frameHeaderSize_min’ defined but not used [-Wunused-const-variable=] 795 | static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN; | ^~~~~~~~~~~~~~~~~~~~~~~~ ./include/linux/zstd.h:794:21: warning: ‘ZSTD_frameHeaderSize_prefix’ defined but not used [-Wunused-const-variable=] 794 | static const size_t ZSTD_frameHeaderSize_prefix = 5; So fix those warnings by turning the constants into defines. Reviewed-by: Nick Terrell Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- include/linux/zstd.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/zstd.h b/include/linux/zstd.h index 249575e2485f..e87f78c9b19c 100644 --- a/include/linux/zstd.h +++ b/include/linux/zstd.h @@ -791,11 +791,11 @@ size_t ZSTD_DStreamOutSize(void); /* for static allocation */ #define ZSTD_FRAMEHEADERSIZE_MAX 18 #define ZSTD_FRAMEHEADERSIZE_MIN 6 -static const size_t ZSTD_frameHeaderSize_prefix = 5; -static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN; -static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +#define ZSTD_frameHeaderSize_prefix 5 +#define ZSTD_frameHeaderSize_min ZSTD_FRAMEHEADERSIZE_MIN +#define ZSTD_frameHeaderSize_max ZSTD_FRAMEHEADERSIZE_MAX /* magic number + skippable frame length */ -static const size_t ZSTD_skippableHeaderSize = 8; +#define ZSTD_skippableHeaderSize 8 /*-************************************* -- cgit v1.2.3 From ae29333fa644679b96d88c9dd3afbef25cbac0f6 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Thu, 4 Feb 2021 19:21:40 +0900 Subject: block: add bio_add_zone_append_page Add bio_add_zone_append_page(), a wrapper around bio_add_hw_page() which is intended to be used by file systems that directly add pages to a bio instead of using bio_iov_iter_get_pages(). Reviewed-by: Christoph Hellwig Reviewed-by: Josef Bacik Reviewed-by: Chaitanya Kulkarni Acked-by: Jens Axboe Signed-off-by: Johannes Thumshirn Signed-off-by: David Sterba --- block/bio.c | 33 +++++++++++++++++++++++++++++++++ include/linux/bio.h | 2 ++ 2 files changed, 35 insertions(+) (limited to 'include/linux') diff --git a/block/bio.c b/block/bio.c index 1f2cc1fbe283..2f21d2958b60 100644 --- a/block/bio.c +++ b/block/bio.c @@ -851,6 +851,39 @@ int bio_add_pc_page(struct request_queue *q, struct bio *bio, } EXPORT_SYMBOL(bio_add_pc_page); +/** + * bio_add_zone_append_page - attempt to add page to zone-append bio + * @bio: destination bio + * @page: page to add + * @len: vec entry length + * @offset: vec entry offset + * + * Attempt to add a page to the bio_vec maplist of a bio that will be submitted + * for a zone-append request. This can fail for a number of reasons, such as the + * bio being full or the target block device is not a zoned block device or + * other limitations of the target block device. The target block device must + * allow bio's up to PAGE_SIZE, so it is always possible to add a single page + * to an empty bio. + * + * Returns: number of bytes added to the bio, or 0 in case of a failure. + */ +int bio_add_zone_append_page(struct bio *bio, struct page *page, + unsigned int len, unsigned int offset) +{ + struct request_queue *q = bio->bi_disk->queue; + bool same_page = false; + + if (WARN_ON_ONCE(bio_op(bio) != REQ_OP_ZONE_APPEND)) + return 0; + + if (WARN_ON_ONCE(!blk_queue_is_zoned(q))) + return 0; + + return bio_add_hw_page(q, bio, page, len, offset, + queue_max_zone_append_sectors(q), &same_page); +} +EXPORT_SYMBOL_GPL(bio_add_zone_append_page); + /** * __bio_try_merge_page - try appending data to an existing bvec. * @bio: destination bio diff --git a/include/linux/bio.h b/include/linux/bio.h index 1edda614f7ce..de62911473bb 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -455,6 +455,8 @@ void bio_chain(struct bio *, struct bio *); extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int); extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *, unsigned int, unsigned int); +int bio_add_zone_append_page(struct bio *bio, struct page *page, + unsigned int len, unsigned int offset); bool __bio_try_merge_page(struct bio *bio, struct page *page, unsigned int len, unsigned int off, bool *same_page); void __bio_add_page(struct bio *bio, struct page *page, -- cgit v1.2.3 From c3b0e880bbfafab6beed92b1ee6db2cdaf4bc54c Mon Sep 17 00:00:00 2001 From: Naohiro Aota Date: Thu, 4 Feb 2021 19:21:41 +0900 Subject: iomap: support REQ_OP_ZONE_APPEND A ZONE_APPEND bio must follow hardware restrictions (e.g. not exceeding max_zone_append_sectors) not to be split. bio_iov_iter_get_pages builds such restricted bio using __bio_iov_append_get_pages if bio_op(bio) == REQ_OP_ZONE_APPEND. To utilize it, we need to set the bio_op before calling bio_iov_iter_get_pages(). This commit introduces IOMAP_F_ZONE_APPEND, so that iomap user can set the flag to indicate they want REQ_OP_ZONE_APPEND and restricted bio. Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Signed-off-by: Naohiro Aota Signed-off-by: David Sterba --- fs/iomap/direct-io.c | 43 +++++++++++++++++++++++++++++++++++++------ include/linux/iomap.h | 1 + 2 files changed, 38 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 933f234d5bec..2273120d8ed7 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -201,6 +201,34 @@ iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos, iomap_dio_submit_bio(dio, iomap, bio, pos); } +/* + * Figure out the bio's operation flags from the dio request, the + * mapping, and whether or not we want FUA. Note that we can end up + * clearing the WRITE_FUA flag in the dio request. + */ +static inline unsigned int +iomap_dio_bio_opflags(struct iomap_dio *dio, struct iomap *iomap, bool use_fua) +{ + unsigned int opflags = REQ_SYNC | REQ_IDLE; + + if (!(dio->flags & IOMAP_DIO_WRITE)) { + WARN_ON_ONCE(iomap->flags & IOMAP_F_ZONE_APPEND); + return REQ_OP_READ; + } + + if (iomap->flags & IOMAP_F_ZONE_APPEND) + opflags |= REQ_OP_ZONE_APPEND; + else + opflags |= REQ_OP_WRITE; + + if (use_fua) + opflags |= REQ_FUA; + else + dio->flags &= ~IOMAP_DIO_WRITE_FUA; + + return opflags; +} + static loff_t iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, struct iomap_dio *dio, struct iomap *iomap) @@ -208,6 +236,7 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, unsigned int blkbits = blksize_bits(bdev_logical_block_size(iomap->bdev)); unsigned int fs_block_size = i_blocksize(inode), pad; unsigned int align = iov_iter_alignment(dio->submit.iter); + unsigned int bio_opf; struct bio *bio; bool need_zeroout = false; bool use_fua = false; @@ -263,6 +292,13 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, iomap_dio_zero(dio, iomap, pos - pad, pad); } + /* + * Set the operation flags early so that bio_iov_iter_get_pages + * can set up the page vector appropriately for a ZONE_APPEND + * operation. + */ + bio_opf = iomap_dio_bio_opflags(dio, iomap, use_fua); + do { size_t n; if (dio->error) { @@ -278,6 +314,7 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, bio->bi_ioprio = dio->iocb->ki_ioprio; bio->bi_private = dio; bio->bi_end_io = iomap_dio_bio_end_io; + bio->bi_opf = bio_opf; ret = bio_iov_iter_get_pages(bio, dio->submit.iter); if (unlikely(ret)) { @@ -293,14 +330,8 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, n = bio->bi_iter.bi_size; if (dio->flags & IOMAP_DIO_WRITE) { - bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; - if (use_fua) - bio->bi_opf |= REQ_FUA; - else - dio->flags &= ~IOMAP_DIO_WRITE_FUA; task_io_account_write(n); } else { - bio->bi_opf = REQ_OP_READ; if (dio->flags & IOMAP_DIO_DIRTY) bio_set_pages_dirty(bio); } diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 5bd3cac4df9c..8ebb1fa6f3b7 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -55,6 +55,7 @@ struct vm_fault; #define IOMAP_F_SHARED 0x04 #define IOMAP_F_MERGED 0x08 #define IOMAP_F_BUFFER_HEAD 0x10 +#define IOMAP_F_ZONE_APPEND 0x20 /* * Flags set by the core iomap code during operations: -- cgit v1.2.3 From 3aa6bce9af0e25b735c9c1263739a5639a336ae8 Mon Sep 17 00:00:00 2001 From: Edwin Peer Date: Fri, 5 Feb 2021 17:37:32 -0800 Subject: net: watchdog: hold device global xmit lock during tx disable Prevent netif_tx_disable() running concurrently with dev_watchdog() by taking the device global xmit lock. Otherwise, the recommended: netif_carrier_off(dev); netif_tx_disable(dev); driver shutdown sequence can happen after the watchdog has already checked carrier, resulting in possible false alarms. This is because netif_tx_lock() only sets the frozen bit without maintaining the locks on the individual queues. Fixes: c3f26a269c24 ("netdev: Fix lockdep warnings in multiqueue configurations.") Signed-off-by: Edwin Peer Reviewed-by: Jakub Kicinski Signed-off-by: David S. Miller --- include/linux/netdevice.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 259be67644e3..5ff27c12ce68 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -4352,6 +4352,7 @@ static inline void netif_tx_disable(struct net_device *dev) local_bh_disable(); cpu = smp_processor_id(); + spin_lock(&dev->tx_global_lock); for (i = 0; i < dev->num_tx_queues; i++) { struct netdev_queue *txq = netdev_get_tx_queue(dev, i); @@ -4359,6 +4360,7 @@ static inline void netif_tx_disable(struct net_device *dev) netif_tx_stop_queue(txq); __netif_tx_unlock(txq); } + spin_unlock(&dev->tx_global_lock); local_bh_enable(); } -- cgit v1.2.3 From 3c5960c0559c44c6628341a82167ee0d3e40ee50 Mon Sep 17 00:00:00 2001 From: Kyle Tso Date: Fri, 5 Feb 2021 11:34:09 +0800 Subject: usb: typec: Manage SVDM version PD Spec Revision 3.0 Version 2.0 + ECNs 2020-12-10 6.4.4.2.3 Structured VDM Version "The Structured VDM Version field of the Discover Identity Command sent and received during VDM discovery Shall be used to determine the lowest common Structured VDM Version supported by the Port Partners or Cable Plug and Shall continue to operate using this Specification Revision until they are Detached." Add a variable in typec_capability to specify the highest SVDM version supported by the port and another variable in typec_partner to cache the negotiated SVDM version between the port and the partner. Also add setter/getter functions for the negotiated SVDM version. Acked-by: Heikki Krogerus Signed-off-by: Kyle Tso Link: https://lore.kernel.org/r/20210205033415.3320439-2-kyletso@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/class.c | 43 +++++++++++++++++++++++++++++++++++++++ include/linux/usb/typec.h | 12 +++++++++++ include/linux/usb/typec_altmode.h | 10 +++++++++ 3 files changed, 65 insertions(+) (limited to 'include/linux') diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index b4a5d9d4564f..45f0bf65e9ab 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -38,6 +38,7 @@ struct typec_partner { struct ida mode_ids; int num_altmodes; u16 pd_revision; /* 0300H = "3.0" */ + enum usb_pd_svdm_ver svdm_version; }; struct typec_port { @@ -824,6 +825,20 @@ typec_partner_register_altmode(struct typec_partner *partner, } EXPORT_SYMBOL_GPL(typec_partner_register_altmode); +/** + * typec_partner_set_svdm_version - Set negotiated Structured VDM (SVDM) Version + * @partner: USB Type-C Partner that supports SVDM + * @svdm_version: Negotiated SVDM Version + * + * This routine is used to save the negotiated SVDM Version. + */ +void typec_partner_set_svdm_version(struct typec_partner *partner, + enum usb_pd_svdm_ver svdm_version) +{ + partner->svdm_version = svdm_version; +} +EXPORT_SYMBOL_GPL(typec_partner_set_svdm_version); + /** * typec_register_partner - Register a USB Type-C Partner * @port: The USB Type-C Port the partner is connected to @@ -848,6 +863,7 @@ struct typec_partner *typec_register_partner(struct typec_port *port, partner->accessory = desc->accessory; partner->num_altmodes = -1; partner->pd_revision = desc->pd_revision; + partner->svdm_version = port->cap->svdm_version; if (desc->identity) { /* @@ -1894,6 +1910,33 @@ EXPORT_SYMBOL_GPL(typec_set_mode); /* --------------------------------------- */ +/** + * typec_get_negotiated_svdm_version - Get negotiated SVDM Version + * @port: USB Type-C Port. + * + * Get the negotiated SVDM Version. The Version is set to the port default + * value stored in typec_capability on partner registration, and updated after + * a successful Discover Identity if the negotiated value is less than the + * default value. + * + * Returns usb_pd_svdm_ver if the partner has been registered otherwise -ENODEV. + */ +int typec_get_negotiated_svdm_version(struct typec_port *port) +{ + enum usb_pd_svdm_ver svdm_version; + struct device *partner_dev; + + partner_dev = device_find_child(&port->dev, NULL, partner_match); + if (!partner_dev) + return -ENODEV; + + svdm_version = to_typec_partner(partner_dev)->svdm_version; + put_device(partner_dev); + + return svdm_version; +} +EXPORT_SYMBOL_GPL(typec_get_negotiated_svdm_version); + /** * typec_get_drvdata - Return private driver data pointer * @port: USB Type-C port diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h index a94df82ab62f..91b4303ca305 100644 --- a/include/linux/usb/typec.h +++ b/include/linux/usb/typec.h @@ -217,12 +217,19 @@ struct typec_operations { enum typec_port_type type); }; +enum usb_pd_svdm_ver { + SVDM_VER_1_0 = 0, + SVDM_VER_2_0 = 1, + SVDM_VER_MAX = SVDM_VER_2_0, +}; + /* * struct typec_capability - USB Type-C Port Capabilities * @type: Supported power role of the port * @data: Supported data role of the port * @revision: USB Type-C Specification release. Binary coded decimal * @pd_revision: USB Power Delivery Specification revision if supported + * @svdm_version: USB PD Structured VDM version if supported * @prefer_role: Initial role preference (DRP ports). * @accessory: Supported Accessory Modes * @fwnode: Optional fwnode of the port @@ -236,6 +243,7 @@ struct typec_capability { enum typec_port_data data; u16 revision; /* 0120H = "1.2" */ u16 pd_revision; /* 0300H = "3.0" */ + enum usb_pd_svdm_ver svdm_version; int prefer_role; enum typec_accessory accessory[TYPEC_MAX_ACCESSORY]; unsigned int orientation_aware:1; @@ -286,4 +294,8 @@ int typec_find_orientation(const char *name); int typec_find_port_power_role(const char *name); int typec_find_power_role(const char *name); int typec_find_port_data_role(const char *name); + +void typec_partner_set_svdm_version(struct typec_partner *partner, + enum usb_pd_svdm_ver svdm_version); +int typec_get_negotiated_svdm_version(struct typec_port *port); #endif /* __LINUX_USB_TYPEC_H */ diff --git a/include/linux/usb/typec_altmode.h b/include/linux/usb/typec_altmode.h index 5e0a7b7647c3..65933cbe9129 100644 --- a/include/linux/usb/typec_altmode.h +++ b/include/linux/usb/typec_altmode.h @@ -132,6 +132,16 @@ typec_altmode_get_orientation(struct typec_altmode *altmode) return typec_get_orientation(typec_altmode2port(altmode)); } +/** + * typec_altmode_get_svdm_version - Get negotiated SVDM version + * @altmode: Handle to the alternate mode + */ +static inline int +typec_altmode_get_svdm_version(struct typec_altmode *altmode) +{ + return typec_get_negotiated_svdm_version(typec_altmode2port(altmode)); +} + /** * struct typec_altmode_driver - USB Type-C alternate mode device driver * @id_table: Null terminated array of SVIDs -- cgit v1.2.3 From 31737c27d665bb3bc8ad9396c63fae2543dd8818 Mon Sep 17 00:00:00 2001 From: Kyle Tso Date: Fri, 5 Feb 2021 11:34:10 +0800 Subject: usb: pd: Make SVDM Version configurable in VDM header PD Rev 3.0 introduces SVDM Version 2.0. This patch makes the field configuable in the header in order to be able to be compatible with older SVDM version. Acked-by: Heikki Krogerus Signed-off-by: Kyle Tso Link: https://lore.kernel.org/r/20210205033415.3320439-3-kyletso@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/altmodes/displayport.c | 2 +- drivers/usb/typec/tcpm/tcpm.c | 16 ++++++++-------- drivers/usb/typec/ucsi/displayport.c | 6 +++--- include/linux/usb/pd_vdo.h | 7 +++++-- 4 files changed, 17 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/typec/altmodes/displayport.c b/drivers/usb/typec/altmodes/displayport.c index e62e5e3da01e..0abc3121238f 100644 --- a/drivers/usb/typec/altmodes/displayport.c +++ b/drivers/usb/typec/altmodes/displayport.c @@ -15,7 +15,7 @@ #include #include "displayport.h" -#define DP_HEADER(_dp, cmd) (VDO((_dp)->alt->svid, 1, cmd) | \ +#define DP_HEADER(_dp, cmd) (VDO((_dp)->alt->svid, 1, SVDM_VER_1_0, cmd) | \ VDO_OPOS(USB_TYPEC_DP_MODE)) enum { diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 8558ab006885..9aadb1e1bec5 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -1544,17 +1544,17 @@ static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev, case CMD_DISCOVER_IDENT: /* 6.4.4.3.1 */ svdm_consume_identity(port, p, cnt); - response[0] = VDO(USB_SID_PD, 1, CMD_DISCOVER_SVID); + response[0] = VDO(USB_SID_PD, 1, SVDM_VER_1_0, CMD_DISCOVER_SVID); rlen = 1; break; case CMD_DISCOVER_SVID: /* 6.4.4.3.2 */ if (svdm_consume_svids(port, p, cnt)) { - response[0] = VDO(USB_SID_PD, 1, + response[0] = VDO(USB_SID_PD, 1, SVDM_VER_1_0, CMD_DISCOVER_SVID); rlen = 1; } else if (modep->nsvids && supports_modal(port)) { - response[0] = VDO(modep->svids[0], 1, + response[0] = VDO(modep->svids[0], 1, SVDM_VER_1_0, CMD_DISCOVER_MODES); rlen = 1; } @@ -1565,7 +1565,7 @@ static int tcpm_pd_svdm(struct tcpm_port *port, struct typec_altmode *adev, modep->svid_index++; if (modep->svid_index < modep->nsvids) { u16 svid = modep->svids[modep->svid_index]; - response[0] = VDO(svid, 1, CMD_DISCOVER_MODES); + response[0] = VDO(svid, 1, SVDM_VER_1_0, CMD_DISCOVER_MODES); rlen = 1; } else { tcpm_register_partner_altmodes(port); @@ -1695,7 +1695,7 @@ static void tcpm_handle_vdm_request(struct tcpm_port *port, break; case ADEV_QUEUE_VDM_SEND_EXIT_MODE_ON_FAIL: if (typec_altmode_vdm(adev, p[0], &p[1], cnt)) { - response[0] = VDO(adev->svid, 1, CMD_EXIT_MODE); + response[0] = VDO(adev->svid, 1, SVDM_VER_1_0, CMD_EXIT_MODE); response[0] |= VDO_OPOS(adev->mode); rlen = 1; } @@ -1729,7 +1729,7 @@ static void tcpm_send_vdm(struct tcpm_port *port, u32 vid, int cmd, /* set VDM header with VID & CMD */ header = VDO(vid, ((vid & USB_SID_PD) == USB_SID_PD) ? - 1 : (PD_VDO_CMD(cmd) <= CMD_ATTENTION), cmd); + 1 : (PD_VDO_CMD(cmd) <= CMD_ATTENTION), SVDM_VER_1_0, cmd); tcpm_queue_vdm(port, header, data, count); } @@ -2024,7 +2024,7 @@ static int tcpm_altmode_enter(struct typec_altmode *altmode, u32 *vdo) struct tcpm_port *port = typec_altmode_get_drvdata(altmode); u32 header; - header = VDO(altmode->svid, vdo ? 2 : 1, CMD_ENTER_MODE); + header = VDO(altmode->svid, vdo ? 2 : 1, SVDM_VER_1_0, CMD_ENTER_MODE); header |= VDO_OPOS(altmode->mode); tcpm_queue_vdm_unlocked(port, header, vdo, vdo ? 1 : 0); @@ -2036,7 +2036,7 @@ static int tcpm_altmode_exit(struct typec_altmode *altmode) struct tcpm_port *port = typec_altmode_get_drvdata(altmode); u32 header; - header = VDO(altmode->svid, 1, CMD_EXIT_MODE); + header = VDO(altmode->svid, 1, SVDM_VER_1_0, CMD_EXIT_MODE); header |= VDO_OPOS(altmode->mode); tcpm_queue_vdm_unlocked(port, header, NULL, 0); diff --git a/drivers/usb/typec/ucsi/displayport.c b/drivers/usb/typec/ucsi/displayport.c index 261131c9e37c..1d387bddefb9 100644 --- a/drivers/usb/typec/ucsi/displayport.c +++ b/drivers/usb/typec/ucsi/displayport.c @@ -83,7 +83,7 @@ static int ucsi_displayport_enter(struct typec_altmode *alt, u32 *vdo) * mode, and letting the alt mode driver continue. */ - dp->header = VDO(USB_TYPEC_DP_SID, 1, CMD_ENTER_MODE); + dp->header = VDO(USB_TYPEC_DP_SID, 1, SVDM_VER_1_0, CMD_ENTER_MODE); dp->header |= VDO_OPOS(USB_TYPEC_DP_MODE); dp->header |= VDO_CMDT(CMDT_RSP_ACK); @@ -120,7 +120,7 @@ static int ucsi_displayport_exit(struct typec_altmode *alt) if (ret < 0) goto out_unlock; - dp->header = VDO(USB_TYPEC_DP_SID, 1, CMD_EXIT_MODE); + dp->header = VDO(USB_TYPEC_DP_SID, 1, SVDM_VER_1_0, CMD_EXIT_MODE); dp->header |= VDO_OPOS(USB_TYPEC_DP_MODE); dp->header |= VDO_CMDT(CMDT_RSP_ACK); @@ -200,7 +200,7 @@ static int ucsi_displayport_vdm(struct typec_altmode *alt, switch (cmd_type) { case CMDT_INIT: - dp->header = VDO(USB_TYPEC_DP_SID, 1, cmd); + dp->header = VDO(USB_TYPEC_DP_SID, 1, SVDM_VER_1_0, cmd); dp->header |= VDO_OPOS(USB_TYPEC_DP_MODE); switch (cmd) { diff --git a/include/linux/usb/pd_vdo.h b/include/linux/usb/pd_vdo.h index 5de7f550f93e..b057250704e8 100644 --- a/include/linux/usb/pd_vdo.h +++ b/include/linux/usb/pd_vdo.h @@ -21,22 +21,24 @@ * ---------- * <31:16> :: SVID * <15> :: VDM type ( 1b == structured, 0b == unstructured ) - * <14:13> :: Structured VDM version (can only be 00 == 1.0 currently) + * <14:13> :: Structured VDM version * <12:11> :: reserved * <10:8> :: object position (1-7 valid ... used for enter/exit mode only) * <7:6> :: command type (SVDM only?) * <5> :: reserved (SVDM), command type (UVDM) * <4:0> :: command */ -#define VDO(vid, type, custom) \ +#define VDO(vid, type, ver, custom) \ (((vid) << 16) | \ ((type) << 15) | \ + ((ver) << 13) | \ ((custom) & 0x7FFF)) #define VDO_SVDM_TYPE (1 << 15) #define VDO_SVDM_VERS(x) ((x) << 13) #define VDO_OPOS(x) ((x) << 8) #define VDO_CMDT(x) ((x) << 6) +#define VDO_SVDM_VERS_MASK VDO_SVDM_VERS(0x3) #define VDO_OPOS_MASK VDO_OPOS(0x7) #define VDO_CMDT_MASK VDO_CMDT(0x3) @@ -74,6 +76,7 @@ #define PD_VDO_VID(vdo) ((vdo) >> 16) #define PD_VDO_SVDM(vdo) (((vdo) >> 15) & 1) +#define PD_VDO_SVDM_VER(vdo) (((vdo) >> 13) & 0x3) #define PD_VDO_OPOS(vdo) (((vdo) >> 8) & 0x7) #define PD_VDO_CMD(vdo) ((vdo) & 0x1f) #define PD_VDO_CMDT(vdo) (((vdo) >> 6) & 0x3) -- cgit v1.2.3 From 9fd6dad1261a541b3f5fa7dc5b152222306e6702 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 5 Feb 2021 05:07:11 -0500 Subject: mm: provide a saner PTE walking API for modules Currently, the follow_pfn function is exported for modules but follow_pte is not. However, follow_pfn is very easy to misuse, because it does not provide protections (so most of its callers assume the page is writable!) and because it returns after having already unlocked the page table lock. Provide instead a simplified version of follow_pte that does not have the pmdpp and range arguments. The older version survives as follow_invalidate_pte() for use by fs/dax.c. Reviewed-by: Jason Gunthorpe Signed-off-by: Paolo Bonzini --- arch/s390/pci/pci_mmio.c | 4 ++-- fs/dax.c | 5 +++-- include/linux/mm.h | 6 ++++-- mm/memory.c | 41 ++++++++++++++++++++++++++++++++++++----- virt/kvm/kvm_main.c | 4 ++-- 5 files changed, 47 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index 18f2d10c3176..474617b88648 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -170,7 +170,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, if (!(vma->vm_flags & VM_WRITE)) goto out_unlock_mmap; - ret = follow_pte(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl); + ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl); if (ret) goto out_unlock_mmap; @@ -311,7 +311,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, if (!(vma->vm_flags & VM_WRITE)) goto out_unlock_mmap; - ret = follow_pte(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl); + ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl); if (ret) goto out_unlock_mmap; diff --git a/fs/dax.c b/fs/dax.c index 26d5dcd2d69e..b3d27fdc6775 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -810,11 +810,12 @@ static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index, address = pgoff_address(index, vma); /* - * Note because we provide range to follow_pte it will call + * follow_invalidate_pte() will use the range to call * mmu_notifier_invalidate_range_start() on our behalf before * taking any lock. */ - if (follow_pte(vma->vm_mm, address, &range, &ptep, &pmdp, &ptl)) + if (follow_invalidate_pte(vma->vm_mm, address, &range, &ptep, + &pmdp, &ptl)) continue; /* diff --git a/include/linux/mm.h b/include/linux/mm.h index ecdf8a8cd6ae..24b292fce8e5 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1658,9 +1658,11 @@ void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling); int copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma); +int follow_invalidate_pte(struct mm_struct *mm, unsigned long address, + struct mmu_notifier_range *range, pte_t **ptepp, + pmd_t **pmdpp, spinlock_t **ptlp); int follow_pte(struct mm_struct *mm, unsigned long address, - struct mmu_notifier_range *range, pte_t **ptepp, pmd_t **pmdpp, - spinlock_t **ptlp); + pte_t **ptepp, spinlock_t **ptlp); int follow_pfn(struct vm_area_struct *vma, unsigned long address, unsigned long *pfn); int follow_phys(struct vm_area_struct *vma, unsigned long address, diff --git a/mm/memory.c b/mm/memory.c index feff48e1465a..985dac0958dc 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4709,9 +4709,9 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) } #endif /* __PAGETABLE_PMD_FOLDED */ -int follow_pte(struct mm_struct *mm, unsigned long address, - struct mmu_notifier_range *range, pte_t **ptepp, pmd_t **pmdpp, - spinlock_t **ptlp) +int follow_invalidate_pte(struct mm_struct *mm, unsigned long address, + struct mmu_notifier_range *range, pte_t **ptepp, + pmd_t **pmdpp, spinlock_t **ptlp) { pgd_t *pgd; p4d_t *p4d; @@ -4776,6 +4776,34 @@ out: return -EINVAL; } +/** + * follow_pte - look up PTE at a user virtual address + * @mm: the mm_struct of the target address space + * @address: user virtual address + * @ptepp: location to store found PTE + * @ptlp: location to store the lock for the PTE + * + * On a successful return, the pointer to the PTE is stored in @ptepp; + * the corresponding lock is taken and its location is stored in @ptlp. + * The contents of the PTE are only stable until @ptlp is released; + * any further use, if any, must be protected against invalidation + * with MMU notifiers. + * + * Only IO mappings and raw PFN mappings are allowed. The mmap semaphore + * should be taken for read. + * + * KVM uses this function. While it is arguably less bad than ``follow_pfn``, + * it is not a good general-purpose API. + * + * Return: zero on success, -ve otherwise. + */ +int follow_pte(struct mm_struct *mm, unsigned long address, + pte_t **ptepp, spinlock_t **ptlp) +{ + return follow_invalidate_pte(mm, address, NULL, ptepp, NULL, ptlp); +} +EXPORT_SYMBOL_GPL(follow_pte); + /** * follow_pfn - look up PFN at a user virtual address * @vma: memory mapping @@ -4784,6 +4812,9 @@ out: * * Only IO mappings and raw PFN mappings are allowed. * + * This function does not allow the caller to read the permissions + * of the PTE. Do not use it. + * * Return: zero and the pfn at @pfn on success, -ve otherwise. */ int follow_pfn(struct vm_area_struct *vma, unsigned long address, @@ -4796,7 +4827,7 @@ int follow_pfn(struct vm_area_struct *vma, unsigned long address, if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) return ret; - ret = follow_pte(vma->vm_mm, address, NULL, &ptep, NULL, &ptl); + ret = follow_pte(vma->vm_mm, address, &ptep, &ptl); if (ret) return ret; *pfn = pte_pfn(*ptep); @@ -4817,7 +4848,7 @@ int follow_phys(struct vm_area_struct *vma, if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) goto out; - if (follow_pte(vma->vm_mm, address, NULL, &ptep, NULL, &ptl)) + if (follow_pte(vma->vm_mm, address, &ptep, &ptl)) goto out; pte = *ptep; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 48ccdf4e3d04..ee4ac2618ec5 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1911,7 +1911,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, spinlock_t *ptl; int r; - r = follow_pte(vma->vm_mm, addr, NULL, &ptep, NULL, &ptl); + r = follow_pte(vma->vm_mm, addr, &ptep, &ptl); if (r) { /* * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does @@ -1926,7 +1926,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, if (r) return r; - r = follow_pte(vma->vm_mm, addr, NULL, &ptep, NULL, &ptl); + r = follow_pte(vma->vm_mm, addr, &ptep, &ptl); if (r) return r; } -- cgit v1.2.3 From c5d1448fa353242635fa3e1fed6ab4558e0e7d9a Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Mon, 8 Feb 2021 15:31:49 +0100 Subject: USB: serial: make remove callback return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All usb_serial drivers return 0 in their remove callbacks and driver core ignores the value returned by usb_serial_device_remove(). So change the remove callback to return void and return 0 unconditionally in usb_serial_device_remove(). Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20210208143149.963644-2-uwe@kleine-koenig.org Signed-off-by: Johan Hovold --- drivers/usb/serial/ark3116.c | 4 +--- drivers/usb/serial/belkin_sa.c | 6 ++---- drivers/usb/serial/bus.c | 5 ++--- drivers/usb/serial/ch341.c | 4 +--- drivers/usb/serial/cp210x.c | 6 ++---- drivers/usb/serial/cyberjack.c | 6 ++---- drivers/usb/serial/cypress_m8.c | 6 ++---- drivers/usb/serial/digi_acceleport.c | 6 ++---- drivers/usb/serial/f81534.c | 3 +-- drivers/usb/serial/ftdi_sio.c | 6 ++---- drivers/usb/serial/garmin_gps.c | 3 +-- drivers/usb/serial/io_edgeport.c | 6 ++---- drivers/usb/serial/io_ti.c | 4 +--- drivers/usb/serial/iuu_phoenix.c | 4 +--- drivers/usb/serial/keyspan.c | 6 ++---- drivers/usb/serial/keyspan_pda.c | 4 +--- drivers/usb/serial/kl5kusb105.c | 6 ++---- drivers/usb/serial/kobil_sct.c | 6 ++---- drivers/usb/serial/mct_u232.c | 6 ++---- drivers/usb/serial/metro-usb.c | 4 +--- drivers/usb/serial/mos7720.c | 4 +--- drivers/usb/serial/mos7840.c | 4 +--- drivers/usb/serial/omninet.c | 6 ++---- drivers/usb/serial/opticon.c | 4 +--- drivers/usb/serial/oti6858.c | 6 ++---- drivers/usb/serial/pl2303.c | 4 +--- drivers/usb/serial/quatech2.c | 4 +--- drivers/usb/serial/sierra.c | 4 +--- drivers/usb/serial/spcp8x5.c | 4 +--- drivers/usb/serial/ssu100.c | 4 +--- drivers/usb/serial/symbolserial.c | 4 +--- drivers/usb/serial/ti_usb_3410_5052.c | 6 ++---- drivers/usb/serial/upd78f0730.c | 4 +--- drivers/usb/serial/usb-wwan.h | 2 +- drivers/usb/serial/usb_wwan.c | 4 +--- drivers/usb/serial/whiteheat.c | 6 ++---- include/linux/usb/serial.h | 2 +- 37 files changed, 53 insertions(+), 120 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/serial/ark3116.c b/drivers/usb/serial/ark3116.c index 71a9206ea1e2..f0ac7bb07ac1 100644 --- a/drivers/usb/serial/ark3116.c +++ b/drivers/usb/serial/ark3116.c @@ -178,15 +178,13 @@ static int ark3116_port_probe(struct usb_serial_port *port) return 0; } -static int ark3116_port_remove(struct usb_serial_port *port) +static void ark3116_port_remove(struct usb_serial_port *port) { struct ark3116_private *priv = usb_get_serial_port_data(port); /* device is closed, so URBs and DMA should be down */ mutex_destroy(&priv->hw_lock); kfree(priv); - - return 0; } static void ark3116_set_termios(struct tty_struct *tty, diff --git a/drivers/usb/serial/belkin_sa.c b/drivers/usb/serial/belkin_sa.c index 9bb123ab9bc9..ed9193f3bb1a 100644 --- a/drivers/usb/serial/belkin_sa.c +++ b/drivers/usb/serial/belkin_sa.c @@ -37,7 +37,7 @@ /* function prototypes for a Belkin USB Serial Adapter F5U103 */ static int belkin_sa_port_probe(struct usb_serial_port *port); -static int belkin_sa_port_remove(struct usb_serial_port *port); +static void belkin_sa_port_remove(struct usb_serial_port *port); static int belkin_sa_open(struct tty_struct *tty, struct usb_serial_port *port); static void belkin_sa_close(struct usb_serial_port *port); @@ -134,14 +134,12 @@ static int belkin_sa_port_probe(struct usb_serial_port *port) return 0; } -static int belkin_sa_port_remove(struct usb_serial_port *port) +static void belkin_sa_port_remove(struct usb_serial_port *port) { struct belkin_sa_private *priv; priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static int belkin_sa_open(struct tty_struct *tty, diff --git a/drivers/usb/serial/bus.c b/drivers/usb/serial/bus.c index 58206ef0a780..1a8c7821d00a 100644 --- a/drivers/usb/serial/bus.c +++ b/drivers/usb/serial/bus.c @@ -88,7 +88,6 @@ static int usb_serial_device_remove(struct device *dev) { struct usb_serial_port *port = to_usb_serial_port(dev); struct usb_serial_driver *driver; - int retval = 0; int minor; int autopm_err; @@ -105,7 +104,7 @@ static int usb_serial_device_remove(struct device *dev) driver = port->serial->type; if (driver->port_remove) - retval = driver->port_remove(port); + driver->port_remove(port); dev_info(dev, "%s converter now disconnected from ttyUSB%d\n", driver->description, minor); @@ -113,7 +112,7 @@ static int usb_serial_device_remove(struct device *dev) if (!autopm_err) usb_autopm_put_interface(port->serial->interface); - return retval; + return 0; } static ssize_t new_id_store(struct device_driver *driver, diff --git a/drivers/usb/serial/ch341.c b/drivers/usb/serial/ch341.c index 28deaaec581f..8d997b71056f 100644 --- a/drivers/usb/serial/ch341.c +++ b/drivers/usb/serial/ch341.c @@ -419,14 +419,12 @@ error: kfree(priv); return r; } -static int ch341_port_remove(struct usb_serial_port *port) +static void ch341_port_remove(struct usb_serial_port *port) { struct ch341_private *priv; priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static int ch341_carrier_raised(struct usb_serial_port *port) diff --git a/drivers/usb/serial/cp210x.c b/drivers/usb/serial/cp210x.c index cc4f63a06f9e..665798043d3c 100644 --- a/drivers/usb/serial/cp210x.c +++ b/drivers/usb/serial/cp210x.c @@ -44,7 +44,7 @@ static int cp210x_attach(struct usb_serial *); static void cp210x_disconnect(struct usb_serial *); static void cp210x_release(struct usb_serial *); static int cp210x_port_probe(struct usb_serial_port *); -static int cp210x_port_remove(struct usb_serial_port *); +static void cp210x_port_remove(struct usb_serial_port *); static void cp210x_dtr_rts(struct usb_serial_port *port, int on); static void cp210x_process_read_urb(struct urb *urb); static void cp210x_enable_event_mode(struct usb_serial_port *port); @@ -1841,14 +1841,12 @@ static int cp210x_port_probe(struct usb_serial_port *port) return 0; } -static int cp210x_port_remove(struct usb_serial_port *port) +static void cp210x_port_remove(struct usb_serial_port *port) { struct cp210x_port_private *port_priv; port_priv = usb_get_serial_port_data(port); kfree(port_priv); - - return 0; } static void cp210x_init_max_speed(struct usb_serial *serial) diff --git a/drivers/usb/serial/cyberjack.c b/drivers/usb/serial/cyberjack.c index 2e40908963da..cf389224d528 100644 --- a/drivers/usb/serial/cyberjack.c +++ b/drivers/usb/serial/cyberjack.c @@ -47,7 +47,7 @@ /* Function prototypes */ static int cyberjack_port_probe(struct usb_serial_port *port); -static int cyberjack_port_remove(struct usb_serial_port *port); +static void cyberjack_port_remove(struct usb_serial_port *port); static int cyberjack_open(struct tty_struct *tty, struct usb_serial_port *port); static void cyberjack_close(struct usb_serial_port *port); @@ -120,7 +120,7 @@ static int cyberjack_port_probe(struct usb_serial_port *port) return 0; } -static int cyberjack_port_remove(struct usb_serial_port *port) +static void cyberjack_port_remove(struct usb_serial_port *port) { struct cyberjack_private *priv; @@ -128,8 +128,6 @@ static int cyberjack_port_remove(struct usb_serial_port *port) priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static int cyberjack_open(struct tty_struct *tty, diff --git a/drivers/usb/serial/cypress_m8.c b/drivers/usb/serial/cypress_m8.c index cc028601c388..166ee2286fda 100644 --- a/drivers/usb/serial/cypress_m8.c +++ b/drivers/usb/serial/cypress_m8.c @@ -115,7 +115,7 @@ struct cypress_private { static int cypress_earthmate_port_probe(struct usb_serial_port *port); static int cypress_hidcom_port_probe(struct usb_serial_port *port); static int cypress_ca42v2_port_probe(struct usb_serial_port *port); -static int cypress_port_remove(struct usb_serial_port *port); +static void cypress_port_remove(struct usb_serial_port *port); static int cypress_open(struct tty_struct *tty, struct usb_serial_port *port); static void cypress_close(struct usb_serial_port *port); static void cypress_dtr_rts(struct usb_serial_port *port, int on); @@ -564,7 +564,7 @@ static int cypress_ca42v2_port_probe(struct usb_serial_port *port) return 0; } -static int cypress_port_remove(struct usb_serial_port *port) +static void cypress_port_remove(struct usb_serial_port *port) { struct cypress_private *priv; @@ -572,8 +572,6 @@ static int cypress_port_remove(struct usb_serial_port *port) kfifo_free(&priv->write_fifo); kfree(priv); - - return 0; } static int cypress_open(struct tty_struct *tty, struct usb_serial_port *port) diff --git a/drivers/usb/serial/digi_acceleport.c b/drivers/usb/serial/digi_acceleport.c index 0ecd5316d85f..8b2f06539f2c 100644 --- a/drivers/usb/serial/digi_acceleport.c +++ b/drivers/usb/serial/digi_acceleport.c @@ -233,7 +233,7 @@ static int digi_startup(struct usb_serial *serial); static void digi_disconnect(struct usb_serial *serial); static void digi_release(struct usb_serial *serial); static int digi_port_probe(struct usb_serial_port *port); -static int digi_port_remove(struct usb_serial_port *port); +static void digi_port_remove(struct usb_serial_port *port); static void digi_read_bulk_callback(struct urb *urb); static int digi_read_inb_callback(struct urb *urb); static int digi_read_oob_callback(struct urb *urb); @@ -1281,14 +1281,12 @@ static int digi_port_probe(struct usb_serial_port *port) return digi_port_init(port, port->port_number); } -static int digi_port_remove(struct usb_serial_port *port) +static void digi_port_remove(struct usb_serial_port *port) { struct digi_port *priv; priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static void digi_read_bulk_callback(struct urb *urb) diff --git a/drivers/usb/serial/f81534.c b/drivers/usb/serial/f81534.c index dd7e55e822ef..a763b362f081 100644 --- a/drivers/usb/serial/f81534.c +++ b/drivers/usb/serial/f81534.c @@ -1430,12 +1430,11 @@ static int f81534_port_probe(struct usb_serial_port *port) return f81534_set_port_output_pin(port); } -static int f81534_port_remove(struct usb_serial_port *port) +static void f81534_port_remove(struct usb_serial_port *port) { struct f81534_port_private *port_priv = usb_get_serial_port_data(port); flush_work(&port_priv->lsr_work); - return 0; } static int f81534_tiocmget(struct tty_struct *tty) diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c index d61703d858a1..c867592477c9 100644 --- a/drivers/usb/serial/ftdi_sio.c +++ b/drivers/usb/serial/ftdi_sio.c @@ -1069,7 +1069,7 @@ static const char *ftdi_chip_name[] = { static int ftdi_sio_probe(struct usb_serial *serial, const struct usb_device_id *id); static int ftdi_sio_port_probe(struct usb_serial_port *port); -static int ftdi_sio_port_remove(struct usb_serial_port *port); +static void ftdi_sio_port_remove(struct usb_serial_port *port); static int ftdi_open(struct tty_struct *tty, struct usb_serial_port *port); static void ftdi_dtr_rts(struct usb_serial_port *port, int on); static void ftdi_process_read_urb(struct urb *urb); @@ -2400,7 +2400,7 @@ static int ftdi_stmclite_probe(struct usb_serial *serial) return 0; } -static int ftdi_sio_port_remove(struct usb_serial_port *port) +static void ftdi_sio_port_remove(struct usb_serial_port *port) { struct ftdi_private *priv = usb_get_serial_port_data(port); @@ -2409,8 +2409,6 @@ static int ftdi_sio_port_remove(struct usb_serial_port *port) remove_sysfs_attrs(port); kfree(priv); - - return 0; } static int ftdi_open(struct tty_struct *tty, struct usb_serial_port *port) diff --git a/drivers/usb/serial/garmin_gps.c b/drivers/usb/serial/garmin_gps.c index c02c19bb1183..50e8bdc77e71 100644 --- a/drivers/usb/serial/garmin_gps.c +++ b/drivers/usb/serial/garmin_gps.c @@ -1401,7 +1401,7 @@ err_free: } -static int garmin_port_remove(struct usb_serial_port *port) +static void garmin_port_remove(struct usb_serial_port *port) { struct garmin_data *garmin_data_p = usb_get_serial_port_data(port); @@ -1409,7 +1409,6 @@ static int garmin_port_remove(struct usb_serial_port *port) usb_kill_urb(port->interrupt_in_urb); del_timer_sync(&garmin_data_p->timer); kfree(garmin_data_p); - return 0; } diff --git a/drivers/usb/serial/io_edgeport.c b/drivers/usb/serial/io_edgeport.c index ba5d8df69518..a493670c06e6 100644 --- a/drivers/usb/serial/io_edgeport.c +++ b/drivers/usb/serial/io_edgeport.c @@ -293,7 +293,7 @@ static int edge_startup(struct usb_serial *serial); static void edge_disconnect(struct usb_serial *serial); static void edge_release(struct usb_serial *serial); static int edge_port_probe(struct usb_serial_port *port); -static int edge_port_remove(struct usb_serial_port *port); +static void edge_port_remove(struct usb_serial_port *port); /* function prototypes for all of our local functions */ @@ -3078,14 +3078,12 @@ static int edge_port_probe(struct usb_serial_port *port) return 0; } -static int edge_port_remove(struct usb_serial_port *port) +static void edge_port_remove(struct usb_serial_port *port) { struct edgeport_port *edge_port; edge_port = usb_get_serial_port_data(port); kfree(edge_port); - - return 0; } static struct usb_serial_driver edgeport_2port_device = { diff --git a/drivers/usb/serial/io_ti.c b/drivers/usb/serial/io_ti.c index 0bbfa47e04b7..e800547be9e0 100644 --- a/drivers/usb/serial/io_ti.c +++ b/drivers/usb/serial/io_ti.c @@ -2625,15 +2625,13 @@ err: return ret; } -static int edge_port_remove(struct usb_serial_port *port) +static void edge_port_remove(struct usb_serial_port *port) { struct edgeport_port *edge_port; edge_port = usb_get_serial_port_data(port); edge_remove_sysfs_attrs(port); kfree(edge_port); - - return 0; } /* Sysfs Attributes */ diff --git a/drivers/usb/serial/iuu_phoenix.c b/drivers/usb/serial/iuu_phoenix.c index e8f06b41a503..093afd67a664 100644 --- a/drivers/usb/serial/iuu_phoenix.c +++ b/drivers/usb/serial/iuu_phoenix.c @@ -100,7 +100,7 @@ static int iuu_port_probe(struct usb_serial_port *port) return 0; } -static int iuu_port_remove(struct usb_serial_port *port) +static void iuu_port_remove(struct usb_serial_port *port) { struct iuu_private *priv = usb_get_serial_port_data(port); @@ -108,8 +108,6 @@ static int iuu_port_remove(struct usb_serial_port *port) kfree(priv->writebuf); kfree(priv->buf); kfree(priv); - - return 0; } static int iuu_tiocmset(struct tty_struct *tty, diff --git a/drivers/usb/serial/keyspan.c b/drivers/usb/serial/keyspan.c index aa3dbce22cfb..622077dcc344 100644 --- a/drivers/usb/serial/keyspan.c +++ b/drivers/usb/serial/keyspan.c @@ -49,7 +49,7 @@ static int keyspan_startup(struct usb_serial *serial); static void keyspan_disconnect(struct usb_serial *serial); static void keyspan_release(struct usb_serial *serial); static int keyspan_port_probe(struct usb_serial_port *port); -static int keyspan_port_remove(struct usb_serial_port *port); +static void keyspan_port_remove(struct usb_serial_port *port); static int keyspan_write_room(struct tty_struct *tty); static int keyspan_write(struct tty_struct *tty, struct usb_serial_port *port, const unsigned char *buf, int count); @@ -2985,7 +2985,7 @@ err_in_buffer: return -ENOMEM; } -static int keyspan_port_remove(struct usb_serial_port *port) +static void keyspan_port_remove(struct usb_serial_port *port) { struct keyspan_port_private *p_priv; int i; @@ -3014,8 +3014,6 @@ static int keyspan_port_remove(struct usb_serial_port *port) kfree(p_priv->in_buffer[i]); kfree(p_priv); - - return 0; } /* Structs for the devices, pre and post renumeration. */ diff --git a/drivers/usb/serial/keyspan_pda.c b/drivers/usb/serial/keyspan_pda.c index e6f933e8d25f..39b0f5f344c2 100644 --- a/drivers/usb/serial/keyspan_pda.c +++ b/drivers/usb/serial/keyspan_pda.c @@ -672,14 +672,12 @@ static int keyspan_pda_port_probe(struct usb_serial_port *port) return 0; } -static int keyspan_pda_port_remove(struct usb_serial_port *port) +static void keyspan_pda_port_remove(struct usb_serial_port *port) { struct keyspan_pda_private *priv; priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static struct usb_serial_driver keyspan_pda_fake_device = { diff --git a/drivers/usb/serial/kl5kusb105.c b/drivers/usb/serial/kl5kusb105.c index 5f6b82ebccc5..f1e9628a9907 100644 --- a/drivers/usb/serial/kl5kusb105.c +++ b/drivers/usb/serial/kl5kusb105.c @@ -52,7 +52,7 @@ * Function prototypes */ static int klsi_105_port_probe(struct usb_serial_port *port); -static int klsi_105_port_remove(struct usb_serial_port *port); +static void klsi_105_port_remove(struct usb_serial_port *port); static int klsi_105_open(struct tty_struct *tty, struct usb_serial_port *port); static void klsi_105_close(struct usb_serial_port *port); static void klsi_105_set_termios(struct tty_struct *tty, @@ -231,14 +231,12 @@ static int klsi_105_port_probe(struct usb_serial_port *port) return 0; } -static int klsi_105_port_remove(struct usb_serial_port *port) +static void klsi_105_port_remove(struct usb_serial_port *port) { struct klsi_105_private *priv; priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static int klsi_105_open(struct tty_struct *tty, struct usb_serial_port *port) diff --git a/drivers/usb/serial/kobil_sct.c b/drivers/usb/serial/kobil_sct.c index 49aacb0a327c..a9bc546626ab 100644 --- a/drivers/usb/serial/kobil_sct.c +++ b/drivers/usb/serial/kobil_sct.c @@ -48,7 +48,7 @@ /* Function prototypes */ static int kobil_port_probe(struct usb_serial_port *probe); -static int kobil_port_remove(struct usb_serial_port *probe); +static void kobil_port_remove(struct usb_serial_port *probe); static int kobil_open(struct tty_struct *tty, struct usb_serial_port *port); static void kobil_close(struct usb_serial_port *port); static int kobil_write(struct tty_struct *tty, struct usb_serial_port *port, @@ -143,14 +143,12 @@ static int kobil_port_probe(struct usb_serial_port *port) } -static int kobil_port_remove(struct usb_serial_port *port) +static void kobil_port_remove(struct usb_serial_port *port) { struct kobil_private *priv; priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static void kobil_init_termios(struct tty_struct *tty) diff --git a/drivers/usb/serial/mct_u232.c b/drivers/usb/serial/mct_u232.c index 7887c312d9a9..ecd5b921e374 100644 --- a/drivers/usb/serial/mct_u232.c +++ b/drivers/usb/serial/mct_u232.c @@ -39,7 +39,7 @@ * Function prototypes */ static int mct_u232_port_probe(struct usb_serial_port *port); -static int mct_u232_port_remove(struct usb_serial_port *remove); +static void mct_u232_port_remove(struct usb_serial_port *remove); static int mct_u232_open(struct tty_struct *tty, struct usb_serial_port *port); static void mct_u232_close(struct usb_serial_port *port); static void mct_u232_dtr_rts(struct usb_serial_port *port, int on); @@ -400,14 +400,12 @@ static int mct_u232_port_probe(struct usb_serial_port *port) return 0; } -static int mct_u232_port_remove(struct usb_serial_port *port) +static void mct_u232_port_remove(struct usb_serial_port *port) { struct mct_u232_private *priv; priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static int mct_u232_open(struct tty_struct *tty, struct usb_serial_port *port) diff --git a/drivers/usb/serial/metro-usb.c b/drivers/usb/serial/metro-usb.c index e63cea02cfd8..0bfe4459c37f 100644 --- a/drivers/usb/serial/metro-usb.c +++ b/drivers/usb/serial/metro-usb.c @@ -256,14 +256,12 @@ static int metrousb_port_probe(struct usb_serial_port *port) return 0; } -static int metrousb_port_remove(struct usb_serial_port *port) +static void metrousb_port_remove(struct usb_serial_port *port) { struct metrousb_private *metro_priv; metro_priv = usb_get_serial_port_data(port); kfree(metro_priv); - - return 0; } static void metrousb_throttle(struct tty_struct *tty) diff --git a/drivers/usb/serial/mos7720.c b/drivers/usb/serial/mos7720.c index aa55169796a3..701dfb32b129 100644 --- a/drivers/usb/serial/mos7720.c +++ b/drivers/usb/serial/mos7720.c @@ -1760,14 +1760,12 @@ static int mos7720_port_probe(struct usb_serial_port *port) return 0; } -static int mos7720_port_remove(struct usb_serial_port *port) +static void mos7720_port_remove(struct usb_serial_port *port) { struct moschip_port *mos7720_port; mos7720_port = usb_get_serial_port_data(port); kfree(mos7720_port); - - return 0; } static struct usb_serial_driver moschip7720_2port_driver = { diff --git a/drivers/usb/serial/mos7840.c b/drivers/usb/serial/mos7840.c index 30c25ef0dacd..1bf0d066f55a 100644 --- a/drivers/usb/serial/mos7840.c +++ b/drivers/usb/serial/mos7840.c @@ -1745,7 +1745,7 @@ error: return status; } -static int mos7840_port_remove(struct usb_serial_port *port) +static void mos7840_port_remove(struct usb_serial_port *port) { struct moschip_port *mos7840_port = usb_get_serial_port_data(port); @@ -1762,8 +1762,6 @@ static int mos7840_port_remove(struct usb_serial_port *port) } kfree(mos7840_port); - - return 0; } static struct usb_serial_driver moschip7840_4port_device = { diff --git a/drivers/usb/serial/omninet.c b/drivers/usb/serial/omninet.c index 5b6e982a9376..83c62f920c50 100644 --- a/drivers/usb/serial/omninet.c +++ b/drivers/usb/serial/omninet.c @@ -36,7 +36,7 @@ static int omninet_prepare_write_buffer(struct usb_serial_port *port, static int omninet_calc_num_ports(struct usb_serial *serial, struct usb_serial_endpoints *epds); static int omninet_port_probe(struct usb_serial_port *port); -static int omninet_port_remove(struct usb_serial_port *port); +static void omninet_port_remove(struct usb_serial_port *port); static const struct usb_device_id id_table[] = { { USB_DEVICE(ZYXEL_VENDOR_ID, ZYXEL_OMNINET_ID) }, @@ -121,14 +121,12 @@ static int omninet_port_probe(struct usb_serial_port *port) return 0; } -static int omninet_port_remove(struct usb_serial_port *port) +static void omninet_port_remove(struct usb_serial_port *port) { struct omninet_data *od; od = usb_get_serial_port_data(port); kfree(od); - - return 0; } #define OMNINET_HEADERLEN 4 diff --git a/drivers/usb/serial/opticon.c b/drivers/usb/serial/opticon.c index 0af76800bd78..eecb72aef83e 100644 --- a/drivers/usb/serial/opticon.c +++ b/drivers/usb/serial/opticon.c @@ -385,13 +385,11 @@ static int opticon_port_probe(struct usb_serial_port *port) return 0; } -static int opticon_port_remove(struct usb_serial_port *port) +static void opticon_port_remove(struct usb_serial_port *port) { struct opticon_private *priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static struct usb_serial_driver opticon_device = { diff --git a/drivers/usb/serial/oti6858.c b/drivers/usb/serial/oti6858.c index 8151dd7a45e8..65cd0341fa78 100644 --- a/drivers/usb/serial/oti6858.c +++ b/drivers/usb/serial/oti6858.c @@ -132,7 +132,7 @@ static int oti6858_tiocmget(struct tty_struct *tty); static int oti6858_tiocmset(struct tty_struct *tty, unsigned int set, unsigned int clear); static int oti6858_port_probe(struct usb_serial_port *port); -static int oti6858_port_remove(struct usb_serial_port *port); +static void oti6858_port_remove(struct usb_serial_port *port); /* device info */ static struct usb_serial_driver oti6858_device = { @@ -344,14 +344,12 @@ static int oti6858_port_probe(struct usb_serial_port *port) return 0; } -static int oti6858_port_remove(struct usb_serial_port *port) +static void oti6858_port_remove(struct usb_serial_port *port) { struct oti6858_private *priv; priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static int oti6858_write(struct tty_struct *tty, struct usb_serial_port *port, diff --git a/drivers/usb/serial/pl2303.c b/drivers/usb/serial/pl2303.c index 29dda60e3bcd..eed9acd1ae08 100644 --- a/drivers/usb/serial/pl2303.c +++ b/drivers/usb/serial/pl2303.c @@ -450,13 +450,11 @@ static int pl2303_port_probe(struct usb_serial_port *port) return 0; } -static int pl2303_port_remove(struct usb_serial_port *port) +static void pl2303_port_remove(struct usb_serial_port *port) { struct pl2303_private *priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static int pl2303_set_control_lines(struct usb_serial_port *port, u8 value) diff --git a/drivers/usb/serial/quatech2.c b/drivers/usb/serial/quatech2.c index 872d1bc86ab4..599dcb2e374d 100644 --- a/drivers/usb/serial/quatech2.c +++ b/drivers/usb/serial/quatech2.c @@ -727,7 +727,7 @@ err_buf: return -ENOMEM; } -static int qt2_port_remove(struct usb_serial_port *port) +static void qt2_port_remove(struct usb_serial_port *port) { struct qt2_port_private *port_priv; @@ -735,8 +735,6 @@ static int qt2_port_remove(struct usb_serial_port *port) usb_free_urb(port_priv->write_urb); kfree(port_priv->write_buffer); kfree(port_priv); - - return 0; } static int qt2_tiocmget(struct tty_struct *tty) diff --git a/drivers/usb/serial/sierra.c b/drivers/usb/serial/sierra.c index 57fc3c31712e..54e16ffc30a0 100644 --- a/drivers/usb/serial/sierra.c +++ b/drivers/usb/serial/sierra.c @@ -901,15 +901,13 @@ static int sierra_port_probe(struct usb_serial_port *port) return 0; } -static int sierra_port_remove(struct usb_serial_port *port) +static void sierra_port_remove(struct usb_serial_port *port) { struct sierra_port_private *portdata; portdata = usb_get_serial_port_data(port); usb_set_serial_port_data(port, NULL); kfree(portdata); - - return 0; } #ifdef CONFIG_PM diff --git a/drivers/usb/serial/spcp8x5.c b/drivers/usb/serial/spcp8x5.c index 3bac55bd9bd9..7039dc918827 100644 --- a/drivers/usb/serial/spcp8x5.c +++ b/drivers/usb/serial/spcp8x5.c @@ -169,14 +169,12 @@ static int spcp8x5_port_probe(struct usb_serial_port *port) return 0; } -static int spcp8x5_port_remove(struct usb_serial_port *port) +static void spcp8x5_port_remove(struct usb_serial_port *port) { struct spcp8x5_private *priv; priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static int spcp8x5_set_ctrl_line(struct usb_serial_port *port, u8 mcr) diff --git a/drivers/usb/serial/ssu100.c b/drivers/usb/serial/ssu100.c index 7d39d35e52a1..89fdc5c19285 100644 --- a/drivers/usb/serial/ssu100.c +++ b/drivers/usb/serial/ssu100.c @@ -366,14 +366,12 @@ static int ssu100_port_probe(struct usb_serial_port *port) return 0; } -static int ssu100_port_remove(struct usb_serial_port *port) +static void ssu100_port_remove(struct usb_serial_port *port) { struct ssu100_port_private *priv; priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static int ssu100_tiocmget(struct tty_struct *tty) diff --git a/drivers/usb/serial/symbolserial.c b/drivers/usb/serial/symbolserial.c index 6ca24e86f686..d7f73ad6e778 100644 --- a/drivers/usb/serial/symbolserial.c +++ b/drivers/usb/serial/symbolserial.c @@ -160,13 +160,11 @@ static int symbol_port_probe(struct usb_serial_port *port) return 0; } -static int symbol_port_remove(struct usb_serial_port *port) +static void symbol_port_remove(struct usb_serial_port *port) { struct symbol_private *priv = usb_get_serial_port_data(port); kfree(priv); - - return 0; } static struct usb_serial_driver symbol_device = { diff --git a/drivers/usb/serial/ti_usb_3410_5052.c b/drivers/usb/serial/ti_usb_3410_5052.c index 73075b9351c5..7252b0ce75a6 100644 --- a/drivers/usb/serial/ti_usb_3410_5052.c +++ b/drivers/usb/serial/ti_usb_3410_5052.c @@ -303,7 +303,7 @@ struct ti_device { static int ti_startup(struct usb_serial *serial); static void ti_release(struct usb_serial *serial); static int ti_port_probe(struct usb_serial_port *port); -static int ti_port_remove(struct usb_serial_port *port); +static void ti_port_remove(struct usb_serial_port *port); static int ti_open(struct tty_struct *tty, struct usb_serial_port *port); static void ti_close(struct usb_serial_port *port); static int ti_write(struct tty_struct *tty, struct usb_serial_port *port, @@ -629,14 +629,12 @@ static int ti_port_probe(struct usb_serial_port *port) return 0; } -static int ti_port_remove(struct usb_serial_port *port) +static void ti_port_remove(struct usb_serial_port *port) { struct ti_port *tport; tport = usb_get_serial_port_data(port); kfree(tport); - - return 0; } static int ti_open(struct tty_struct *tty, struct usb_serial_port *port) diff --git a/drivers/usb/serial/upd78f0730.c b/drivers/usb/serial/upd78f0730.c index 1ca9c1881621..26d7b003b7e3 100644 --- a/drivers/usb/serial/upd78f0730.c +++ b/drivers/usb/serial/upd78f0730.c @@ -171,15 +171,13 @@ static int upd78f0730_port_probe(struct usb_serial_port *port) return 0; } -static int upd78f0730_port_remove(struct usb_serial_port *port) +static void upd78f0730_port_remove(struct usb_serial_port *port) { struct upd78f0730_port_private *private; private = usb_get_serial_port_data(port); mutex_destroy(&private->lock); kfree(private); - - return 0; } static int upd78f0730_tiocmget(struct tty_struct *tty) diff --git a/drivers/usb/serial/usb-wwan.h b/drivers/usb/serial/usb-wwan.h index 934e9361cf6b..79dafd98e0a1 100644 --- a/drivers/usb/serial/usb-wwan.h +++ b/drivers/usb/serial/usb-wwan.h @@ -10,7 +10,7 @@ extern void usb_wwan_dtr_rts(struct usb_serial_port *port, int on); extern int usb_wwan_open(struct tty_struct *tty, struct usb_serial_port *port); extern void usb_wwan_close(struct usb_serial_port *port); extern int usb_wwan_port_probe(struct usb_serial_port *port); -extern int usb_wwan_port_remove(struct usb_serial_port *port); +extern void usb_wwan_port_remove(struct usb_serial_port *port); extern int usb_wwan_write_room(struct tty_struct *tty); extern int usb_wwan_tiocmget(struct tty_struct *tty); extern int usb_wwan_tiocmset(struct tty_struct *tty, diff --git a/drivers/usb/serial/usb_wwan.c b/drivers/usb/serial/usb_wwan.c index 4b9845807bee..46d46a4f99c9 100644 --- a/drivers/usb/serial/usb_wwan.c +++ b/drivers/usb/serial/usb_wwan.c @@ -544,7 +544,7 @@ bail_out_error: } EXPORT_SYMBOL_GPL(usb_wwan_port_probe); -int usb_wwan_port_remove(struct usb_serial_port *port) +void usb_wwan_port_remove(struct usb_serial_port *port) { int i; struct usb_wwan_port_private *portdata; @@ -562,8 +562,6 @@ int usb_wwan_port_remove(struct usb_serial_port *port) } kfree(portdata); - - return 0; } EXPORT_SYMBOL(usb_wwan_port_remove); diff --git a/drivers/usb/serial/whiteheat.c b/drivers/usb/serial/whiteheat.c index ca3bd58f2025..ccfd5ed652cd 100644 --- a/drivers/usb/serial/whiteheat.c +++ b/drivers/usb/serial/whiteheat.c @@ -79,7 +79,7 @@ static int whiteheat_firmware_attach(struct usb_serial *serial); static int whiteheat_attach(struct usb_serial *serial); static void whiteheat_release(struct usb_serial *serial); static int whiteheat_port_probe(struct usb_serial_port *port); -static int whiteheat_port_remove(struct usb_serial_port *port); +static void whiteheat_port_remove(struct usb_serial_port *port); static int whiteheat_open(struct tty_struct *tty, struct usb_serial_port *port); static void whiteheat_close(struct usb_serial_port *port); @@ -345,14 +345,12 @@ static int whiteheat_port_probe(struct usb_serial_port *port) return 0; } -static int whiteheat_port_remove(struct usb_serial_port *port) +static void whiteheat_port_remove(struct usb_serial_port *port) { struct whiteheat_private *info; info = usb_get_serial_port_data(port); kfree(info); - - return 0; } static int whiteheat_open(struct tty_struct *tty, struct usb_serial_port *port) diff --git a/include/linux/usb/serial.h b/include/linux/usb/serial.h index 1c09b922f8b0..952272002e48 100644 --- a/include/linux/usb/serial.h +++ b/include/linux/usb/serial.h @@ -260,7 +260,7 @@ struct usb_serial_driver { void (*release)(struct usb_serial *serial); int (*port_probe)(struct usb_serial_port *port); - int (*port_remove)(struct usb_serial_port *port); + void (*port_remove)(struct usb_serial_port *port); int (*suspend)(struct usb_serial *serial, pm_message_t message); int (*resume)(struct usb_serial *serial); -- cgit v1.2.3 From 4fc096a99e01dd06dc55bef76ade7f8d76653245 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Thu, 28 Jan 2021 13:01:31 -0500 Subject: KVM: Raise the maximum number of user memslots Current KVM_USER_MEM_SLOTS limits are arch specific (512 on Power, 509 on x86, 32 on s390, 16 on MIPS) but they don't really need to be. Memory slots are allocated dynamically in KVM when added so the only real limitation is 'id_to_index' array which is 'short'. We don't have any other KVM_MEM_SLOTS_NUM/KVM_USER_MEM_SLOTS-sized statically defined structures. Low KVM_USER_MEM_SLOTS can be a limiting factor for some configurations. In particular, when QEMU tries to start a Windows guest with Hyper-V SynIC enabled and e.g. 256 vCPUs the limit is hit as SynIC requires two pages per vCPU and the guest is free to pick any GFN for each of them, this fragments memslots as QEMU wants to have a separate memslot for each of these pages (which are supposed to act as 'overlay' pages). Signed-off-by: Vitaly Kuznetsov Message-Id: <20210127175731.2020089-3-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- arch/arm64/include/asm/kvm_host.h | 1 - arch/mips/include/asm/kvm_host.h | 1 - arch/powerpc/include/asm/kvm_host.h | 1 - arch/s390/include/asm/kvm_host.h | 1 - arch/x86/include/asm/kvm_host.h | 2 -- include/linux/kvm_host.h | 5 ++--- 6 files changed, 2 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 8fcfab0c2567..1b8a3d825276 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -30,7 +30,6 @@ #define __KVM_HAVE_ARCH_INTC_INITIALIZED -#define KVM_USER_MEM_SLOTS 512 #define KVM_HALT_POLL_NS_DEFAULT 500000 #include diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h index 24f3d0f9996b..3a5612e7304c 100644 --- a/arch/mips/include/asm/kvm_host.h +++ b/arch/mips/include/asm/kvm_host.h @@ -83,7 +83,6 @@ #define KVM_MAX_VCPUS 16 -#define KVM_USER_MEM_SLOTS 16 /* memory slots that does not exposed to userspace */ #define KVM_PRIVATE_MEM_SLOTS 0 diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index d67a470e95a3..2b9b6855ec86 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -28,7 +28,6 @@ #define KVM_MAX_VCPUS NR_CPUS #define KVM_MAX_VCORES NR_CPUS -#define KVM_USER_MEM_SLOTS 512 #include diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 74f9a036bab2..6bcfc5614bbc 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -28,7 +28,6 @@ #define KVM_S390_BSCA_CPU_SLOTS 64 #define KVM_S390_ESCA_CPU_SLOTS 248 #define KVM_MAX_VCPUS 255 -#define KVM_USER_MEM_SLOTS 32 /* * These seem to be used for allocating ->chip in the routing table, which we diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index c451412a6230..b6de568e7476 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -40,10 +40,8 @@ #define KVM_MAX_VCPUS 288 #define KVM_SOFT_MAX_VCPUS 240 #define KVM_MAX_VCPU_ID 1023 -#define KVM_USER_MEM_SLOTS 509 /* memory slots that are not exposed to userspace */ #define KVM_PRIVATE_MEM_SLOTS 3 -#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS) #define KVM_HALT_POLL_NS_DEFAULT 200000 diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index f417447129b9..e126ebda36d0 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -425,9 +425,8 @@ struct kvm_irq_routing_table { #define KVM_PRIVATE_MEM_SLOTS 0 #endif -#ifndef KVM_MEM_SLOTS_NUM -#define KVM_MEM_SLOTS_NUM (KVM_USER_MEM_SLOTS + KVM_PRIVATE_MEM_SLOTS) -#endif +#define KVM_MEM_SLOTS_NUM SHRT_MAX +#define KVM_USER_MEM_SLOTS (KVM_MEM_SLOTS_NUM - KVM_PRIVATE_MEM_SLOTS) #ifndef __KVM_VCPU_MULTIPLE_ADDRESS_SPACE static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From 29863d41bb6e1d969c62fdb15b0961806942960e Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Mon, 8 Feb 2021 11:34:09 -0800 Subject: net: implement threaded-able napi poll loop support This patch allows running each napi poll loop inside its own kernel thread. The kthread is created during netif_napi_add() if dev->threaded is set. And threaded mode is enabled in napi_enable(). We will provide a way to set dev->threaded and enable threaded mode without a device up/down in the following patch. Once that threaded mode is enabled and the kthread is started, napi_schedule() will wake-up such thread instead of scheduling the softirq. The threaded poll loop behaves quite likely the net_rx_action, but it does not have to manipulate local irqs and uses an explicit scheduling point based on netdev_budget. Co-developed-by: Paolo Abeni Signed-off-by: Paolo Abeni Co-developed-by: Hannes Frederic Sowa Signed-off-by: Hannes Frederic Sowa Co-developed-by: Jakub Kicinski Signed-off-by: Jakub Kicinski Signed-off-by: Wei Wang Reviewed-by: Alexander Duyck Signed-off-by: David S. Miller --- include/linux/netdevice.h | 21 +++------ net/core/dev.c | 112 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index e9e7ada07ea1..99fb4ec9573e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -347,6 +347,7 @@ struct napi_struct { struct list_head dev_list; struct hlist_node napi_hash_node; unsigned int napi_id; + struct task_struct *thread; }; enum { @@ -358,6 +359,7 @@ enum { NAPI_STATE_NO_BUSY_POLL, /* Do not add in napi_hash, no busy polling */ NAPI_STATE_IN_BUSY_POLL, /* sk_busy_loop() owns this NAPI */ NAPI_STATE_PREFER_BUSY_POLL, /* prefer busy-polling over softirq processing*/ + NAPI_STATE_THREADED, /* The poll is performed inside its own thread*/ }; enum { @@ -369,6 +371,7 @@ enum { NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL), NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL), NAPIF_STATE_PREFER_BUSY_POLL = BIT(NAPI_STATE_PREFER_BUSY_POLL), + NAPIF_STATE_THREADED = BIT(NAPI_STATE_THREADED), }; enum gro_result { @@ -503,20 +506,7 @@ static inline bool napi_complete(struct napi_struct *n) */ void napi_disable(struct napi_struct *n); -/** - * napi_enable - enable NAPI scheduling - * @n: NAPI context - * - * Resume NAPI from being scheduled on this context. - * Must be paired with napi_disable. - */ -static inline void napi_enable(struct napi_struct *n) -{ - BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); - smp_mb__before_atomic(); - clear_bit(NAPI_STATE_SCHED, &n->state); - clear_bit(NAPI_STATE_NPSVC, &n->state); -} +void napi_enable(struct napi_struct *n); /** * napi_synchronize - wait until NAPI is not running @@ -1827,6 +1817,8 @@ enum netdev_priv_flags { * * @wol_enabled: Wake-on-LAN is enabled * + * @threaded: napi threaded mode is enabled + * * @net_notifier_list: List of per-net netdev notifier block * that follow this device when it is moved * to another network namespace. @@ -2145,6 +2137,7 @@ struct net_device { struct lock_class_key *qdisc_running_key; bool proto_down; unsigned wol_enabled:1; + unsigned threaded:1; struct list_head net_notifier_list; diff --git a/net/core/dev.c b/net/core/dev.c index 59751a22d7c3..1e35f4f44f3b 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -91,6 +91,7 @@ #include #include #include +#include #include #include #include @@ -1494,6 +1495,27 @@ void netdev_notify_peers(struct net_device *dev) } EXPORT_SYMBOL(netdev_notify_peers); +static int napi_threaded_poll(void *data); + +static int napi_kthread_create(struct napi_struct *n) +{ + int err = 0; + + /* Create and wake up the kthread once to put it in + * TASK_INTERRUPTIBLE mode to avoid the blocked task + * warning and work with loadavg. + */ + n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d", + n->dev->name, n->napi_id); + if (IS_ERR(n->thread)) { + err = PTR_ERR(n->thread); + pr_err("kthread_run failed with err %d\n", err); + n->thread = NULL; + } + + return err; +} + static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack) { const struct net_device_ops *ops = dev->netdev_ops; @@ -4265,6 +4287,21 @@ int gro_normal_batch __read_mostly = 8; static inline void ____napi_schedule(struct softnet_data *sd, struct napi_struct *napi) { + struct task_struct *thread; + + if (test_bit(NAPI_STATE_THREADED, &napi->state)) { + /* Paired with smp_mb__before_atomic() in + * napi_enable(). Use READ_ONCE() to guarantee + * a complete read on napi->thread. Only call + * wake_up_process() when it's not NULL. + */ + thread = READ_ONCE(napi->thread); + if (thread) { + wake_up_process(thread); + return; + } + } + list_add_tail(&napi->poll_list, &sd->poll_list); __raise_softirq_irqoff(NET_RX_SOFTIRQ); } @@ -6728,6 +6765,12 @@ void netif_napi_add(struct net_device *dev, struct napi_struct *napi, set_bit(NAPI_STATE_NPSVC, &napi->state); list_add_rcu(&napi->dev_list, &dev->napi_list); napi_hash_add(napi); + /* Create kthread for this napi if dev->threaded is set. + * Clear dev->threaded if kthread creation failed so that + * threaded mode will not be enabled in napi_enable(). + */ + if (dev->threaded && napi_kthread_create(napi)) + dev->threaded = 0; } EXPORT_SYMBOL(netif_napi_add); @@ -6745,9 +6788,28 @@ void napi_disable(struct napi_struct *n) clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &n->state); clear_bit(NAPI_STATE_DISABLE, &n->state); + clear_bit(NAPI_STATE_THREADED, &n->state); } EXPORT_SYMBOL(napi_disable); +/** + * napi_enable - enable NAPI scheduling + * @n: NAPI context + * + * Resume NAPI from being scheduled on this context. + * Must be paired with napi_disable. + */ +void napi_enable(struct napi_struct *n) +{ + BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); + smp_mb__before_atomic(); + clear_bit(NAPI_STATE_SCHED, &n->state); + clear_bit(NAPI_STATE_NPSVC, &n->state); + if (n->dev->threaded && n->thread) + set_bit(NAPI_STATE_THREADED, &n->state); +} +EXPORT_SYMBOL(napi_enable); + static void flush_gro_hash(struct napi_struct *napi) { int i; @@ -6773,6 +6835,11 @@ void __netif_napi_del(struct napi_struct *napi) flush_gro_hash(napi); napi->gro_bitmask = 0; + + if (napi->thread) { + kthread_stop(napi->thread); + napi->thread = NULL; + } } EXPORT_SYMBOL(__netif_napi_del); @@ -6867,6 +6934,51 @@ static int napi_poll(struct napi_struct *n, struct list_head *repoll) return work; } +static int napi_thread_wait(struct napi_struct *napi) +{ + set_current_state(TASK_INTERRUPTIBLE); + + while (!kthread_should_stop() && !napi_disable_pending(napi)) { + if (test_bit(NAPI_STATE_SCHED, &napi->state)) { + WARN_ON(!list_empty(&napi->poll_list)); + __set_current_state(TASK_RUNNING); + return 0; + } + + schedule(); + set_current_state(TASK_INTERRUPTIBLE); + } + __set_current_state(TASK_RUNNING); + return -1; +} + +static int napi_threaded_poll(void *data) +{ + struct napi_struct *napi = data; + void *have; + + while (!napi_thread_wait(napi)) { + for (;;) { + bool repoll = false; + + local_bh_disable(); + + have = netpoll_poll_lock(napi); + __napi_poll(napi, &repoll); + netpoll_poll_unlock(have); + + __kfree_skb_flush(); + local_bh_enable(); + + if (!repoll) + break; + + cond_resched(); + } + } + return 0; +} + static __latent_entropy void net_rx_action(struct softirq_action *h) { struct softnet_data *sd = this_cpu_ptr(&softnet_data); -- cgit v1.2.3 From 5fdd2f0e5c64846bf3066689b73fc3b8dddd1c74 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Mon, 8 Feb 2021 11:34:10 -0800 Subject: net: add sysfs attribute to control napi threaded mode This patch adds a new sysfs attribute to the network device class. Said attribute provides a per-device control to enable/disable the threaded mode for all the napi instances of the given network device, without the need for a device up/down. User sets it to 1 or 0 to enable or disable threaded mode. Note: when switching between threaded and the current softirq based mode for a napi instance, it will not immediately take effect if the napi is currently being polled. The mode switch will happen for the next time napi_schedule() is called. Co-developed-by: Paolo Abeni Signed-off-by: Paolo Abeni Co-developed-by: Hannes Frederic Sowa Signed-off-by: Hannes Frederic Sowa Co-developed-by: Felix Fietkau Signed-off-by: Felix Fietkau Signed-off-by: Wei Wang Reviewed-by: Alexander Duyck Signed-off-by: David S. Miller --- Documentation/ABI/testing/sysfs-class-net | 15 ++++++++++ include/linux/netdevice.h | 2 ++ net/core/dev.c | 48 +++++++++++++++++++++++++++++-- net/core/net-sysfs.c | 40 ++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/Documentation/ABI/testing/sysfs-class-net b/Documentation/ABI/testing/sysfs-class-net index 1f2002df5ba2..1419103d11f9 100644 --- a/Documentation/ABI/testing/sysfs-class-net +++ b/Documentation/ABI/testing/sysfs-class-net @@ -337,3 +337,18 @@ Contact: netdev@vger.kernel.org Description: 32-bit unsigned integer counting the number of times the link has been down + +What: /sys/class/net//threaded +Date: Jan 2021 +KernelVersion: 5.12 +Contact: netdev@vger.kernel.org +Description: + Boolean value to control the threaded mode per device. User could + set this value to enable/disable threaded mode for all napi + belonging to this device, without the need to do device up/down. + + Possible values: + == ================================== + 0 threaded mode disabled for this dev + 1 threaded mode enabled for this dev + == ================================== diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 99fb4ec9573e..1340327f7abf 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -497,6 +497,8 @@ static inline bool napi_complete(struct napi_struct *n) return napi_complete_done(n, 0); } +int dev_set_threaded(struct net_device *dev, bool threaded); + /** * napi_disable - prevent NAPI from scheduling * @n: NAPI context diff --git a/net/core/dev.c b/net/core/dev.c index 1e35f4f44f3b..7647278e46f0 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4291,8 +4291,9 @@ static inline void ____napi_schedule(struct softnet_data *sd, if (test_bit(NAPI_STATE_THREADED, &napi->state)) { /* Paired with smp_mb__before_atomic() in - * napi_enable(). Use READ_ONCE() to guarantee - * a complete read on napi->thread. Only call + * napi_enable()/dev_set_threaded(). + * Use READ_ONCE() to guarantee a complete + * read on napi->thread. Only call * wake_up_process() when it's not NULL. */ thread = READ_ONCE(napi->thread); @@ -6738,6 +6739,49 @@ static void init_gro_hash(struct napi_struct *napi) napi->gro_bitmask = 0; } +int dev_set_threaded(struct net_device *dev, bool threaded) +{ + struct napi_struct *napi; + int err = 0; + + if (dev->threaded == threaded) + return 0; + + if (threaded) { + list_for_each_entry(napi, &dev->napi_list, dev_list) { + if (!napi->thread) { + err = napi_kthread_create(napi); + if (err) { + threaded = false; + break; + } + } + } + } + + dev->threaded = threaded; + + /* Make sure kthread is created before THREADED bit + * is set. + */ + smp_mb__before_atomic(); + + /* Setting/unsetting threaded mode on a napi might not immediately + * take effect, if the current napi instance is actively being + * polled. In this case, the switch between threaded mode and + * softirq mode will happen in the next round of napi_schedule(). + * This should not cause hiccups/stalls to the live traffic. + */ + list_for_each_entry(napi, &dev->napi_list, dev_list) { + if (threaded) + set_bit(NAPI_STATE_THREADED, &napi->state); + else + clear_bit(NAPI_STATE_THREADED, &napi->state); + } + + return err; +} + void netif_napi_add(struct net_device *dev, struct napi_struct *napi, int (*poll)(struct napi_struct *, int), int weight) { diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index 91afb0b6de69..307628fdf380 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -538,6 +538,45 @@ static ssize_t phys_switch_id_show(struct device *dev, } static DEVICE_ATTR_RO(phys_switch_id); +static ssize_t threaded_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct net_device *netdev = to_net_dev(dev); + ssize_t ret = -EINVAL; + + if (!rtnl_trylock()) + return restart_syscall(); + + if (dev_isalive(netdev)) + ret = sprintf(buf, fmt_dec, netdev->threaded); + + rtnl_unlock(); + return ret; +} + +static int modify_napi_threaded(struct net_device *dev, unsigned long val) +{ + int ret; + + if (list_empty(&dev->napi_list)) + return -EOPNOTSUPP; + + if (val != 0 && val != 1) + return -EOPNOTSUPP; + + ret = dev_set_threaded(dev, val); + + return ret; +} + +static ssize_t threaded_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + return netdev_store(dev, attr, buf, len, modify_napi_threaded); +} +static DEVICE_ATTR_RW(threaded); + static struct attribute *net_class_attrs[] __ro_after_init = { &dev_attr_netdev_group.attr, &dev_attr_type.attr, @@ -570,6 +609,7 @@ static struct attribute *net_class_attrs[] __ro_after_init = { &dev_attr_proto_down.attr, &dev_attr_carrier_up_count.attr, &dev_attr_carrier_down_count.attr, + &dev_attr_threaded.attr, NULL, }; ATTRIBUTE_GROUPS(net_class); -- cgit v1.2.3 From 0f319d49a4167e402b01b2b56639386f0b6846ba Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 10 Feb 2021 09:52:47 +0100 Subject: locking/mutex: Kill mutex_trylock_recursive() There are not users of mutex_trylock_recursive() in tree as of v5.11-rc7. Remove it. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20210210085248.219210-2-bigeasy@linutronix.de --- include/linux/mutex.h | 25 ------------------------- kernel/locking/mutex.c | 10 ---------- 2 files changed, 35 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mutex.h b/include/linux/mutex.h index dcd185cbfe79..0cd631a19727 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h @@ -199,29 +199,4 @@ extern void mutex_unlock(struct mutex *lock); extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock); -/* - * These values are chosen such that FAIL and SUCCESS match the - * values of the regular mutex_trylock(). - */ -enum mutex_trylock_recursive_enum { - MUTEX_TRYLOCK_FAILED = 0, - MUTEX_TRYLOCK_SUCCESS = 1, - MUTEX_TRYLOCK_RECURSIVE, -}; - -/** - * mutex_trylock_recursive - trylock variant that allows recursive locking - * @lock: mutex to be locked - * - * This function should not be used, _ever_. It is purely for hysterical GEM - * raisins, and once those are gone this will be removed. - * - * Returns: - * - MUTEX_TRYLOCK_FAILED - trylock failed, - * - MUTEX_TRYLOCK_SUCCESS - lock acquired, - * - MUTEX_TRYLOCK_RECURSIVE - we already owned the lock. - */ -extern /* __deprecated */ __must_check enum mutex_trylock_recursive_enum -mutex_trylock_recursive(struct mutex *lock); - #endif /* __LINUX_MUTEX_H */ diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 5352ce50a97e..adb935090768 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -86,16 +86,6 @@ bool mutex_is_locked(struct mutex *lock) } EXPORT_SYMBOL(mutex_is_locked); -__must_check enum mutex_trylock_recursive_enum -mutex_trylock_recursive(struct mutex *lock) -{ - if (unlikely(__mutex_owner(lock) == current)) - return MUTEX_TRYLOCK_RECURSIVE; - - return mutex_trylock(lock); -} -EXPORT_SYMBOL(mutex_trylock_recursive); - static inline unsigned long __owner_flags(unsigned long owner) { return owner & MUTEX_FLAGS; -- cgit v1.2.3 From 7cbf1722d5fc5779946ee8f338e9e38b5de15856 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 10 Feb 2021 00:03:20 +0000 Subject: io_uring: provide FIFO ordering for task_work task_work is a LIFO list, due to how it's implemented as a lockless list. For long chains of task_work, this can be problematic as the first entry added is the last one processed. Similarly, we'd waste a lot of CPU cycles reversing this list. Wrap the task_work so we have a single task_work entry per task per ctx, and use that to run it in the right order. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io-wq.h | 9 ----- fs/io_uring.c | 101 +++++++++++++++++++++++++++++++++++++++++++---- include/linux/io_uring.h | 14 +++++++ 3 files changed, 108 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/fs/io-wq.h b/fs/io-wq.h index e37a0f217cc8..096f1021018e 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -27,15 +27,6 @@ enum io_wq_cancel { IO_WQ_CANCEL_NOTFOUND, /* work not found */ }; -struct io_wq_work_node { - struct io_wq_work_node *next; -}; - -struct io_wq_work_list { - struct io_wq_work_node *first; - struct io_wq_work_node *last; -}; - static inline void wq_list_add_after(struct io_wq_work_node *node, struct io_wq_work_node *pos, struct io_wq_work_list *list) diff --git a/fs/io_uring.c b/fs/io_uring.c index 87a4b727fe1c..bfc8fcd93504 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -721,6 +721,11 @@ struct async_poll { struct io_poll_iocb *double_poll; }; +struct io_task_work { + struct io_wq_work_node node; + task_work_func_t func; +}; + /* * NOTE! Each of the iocb union members has the file pointer * as the first entry in their struct definition. So you can @@ -779,7 +784,10 @@ struct io_kiocb { * 2. to track reqs with ->files (see io_op_def::file_table) */ struct list_head inflight_entry; - struct callback_head task_work; + union { + struct io_task_work io_task_work; + struct callback_head task_work; + }; /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */ struct hlist_node hash_node; struct async_poll *apoll; @@ -2129,6 +2137,81 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req) return __io_req_find_next(req); } +static bool __tctx_task_work(struct io_uring_task *tctx) +{ + struct io_wq_work_list list; + struct io_wq_work_node *node; + + if (wq_list_empty(&tctx->task_list)) + return false; + + spin_lock(&tctx->task_lock); + list = tctx->task_list; + INIT_WQ_LIST(&tctx->task_list); + spin_unlock(&tctx->task_lock); + + node = list.first; + while (node) { + struct io_wq_work_node *next = node->next; + struct io_kiocb *req; + + req = container_of(node, struct io_kiocb, io_task_work.node); + req->task_work.func(&req->task_work); + node = next; + } + + return list.first != NULL; +} + +static void tctx_task_work(struct callback_head *cb) +{ + struct io_uring_task *tctx = container_of(cb, struct io_uring_task, task_work); + + while (__tctx_task_work(tctx)) + cond_resched(); + + clear_bit(0, &tctx->task_state); +} + +static int io_task_work_add(struct task_struct *tsk, struct io_kiocb *req, + enum task_work_notify_mode notify) +{ + struct io_uring_task *tctx = tsk->io_uring; + struct io_wq_work_node *node, *prev; + int ret; + + WARN_ON_ONCE(!tctx); + + spin_lock(&tctx->task_lock); + wq_list_add_tail(&req->io_task_work.node, &tctx->task_list); + spin_unlock(&tctx->task_lock); + + /* task_work already pending, we're done */ + if (test_bit(0, &tctx->task_state) || + test_and_set_bit(0, &tctx->task_state)) + return 0; + + if (!task_work_add(tsk, &tctx->task_work, notify)) + return 0; + + /* + * Slow path - we failed, find and delete work. if the work is not + * in the list, it got run and we're fine. + */ + ret = 0; + spin_lock(&tctx->task_lock); + wq_list_for_each(node, prev, &tctx->task_list) { + if (&req->io_task_work.node == node) { + wq_list_del(&tctx->task_list, node, prev); + ret = 1; + break; + } + } + spin_unlock(&tctx->task_lock); + clear_bit(0, &tctx->task_state); + return ret; +} + static int io_req_task_work_add(struct io_kiocb *req) { struct task_struct *tsk = req->task; @@ -2149,7 +2232,7 @@ static int io_req_task_work_add(struct io_kiocb *req) if (!(ctx->flags & IORING_SETUP_SQPOLL)) notify = TWA_SIGNAL; - ret = task_work_add(tsk, &req->task_work, notify); + ret = io_task_work_add(tsk, req, notify); if (!ret) wake_up_process(tsk); @@ -2157,7 +2240,7 @@ static int io_req_task_work_add(struct io_kiocb *req) } static void io_req_task_work_add_fallback(struct io_kiocb *req, - void (*cb)(struct callback_head *)) + task_work_func_t cb) { struct task_struct *tsk = io_wq_get_task(req->ctx->io_wq); @@ -2216,7 +2299,7 @@ static void io_req_task_queue(struct io_kiocb *req) { int ret; - init_task_work(&req->task_work, io_req_task_submit); + req->task_work.func = io_req_task_submit; percpu_ref_get(&req->ctx->refs); ret = io_req_task_work_add(req); @@ -2347,7 +2430,7 @@ static void io_free_req_deferred(struct io_kiocb *req) { int ret; - init_task_work(&req->task_work, io_put_req_deferred_cb); + req->task_work.func = io_put_req_deferred_cb; ret = io_req_task_work_add(req); if (unlikely(ret)) io_req_task_work_add_fallback(req, io_put_req_deferred_cb); @@ -3392,7 +3475,7 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, req->rw.kiocb.ki_flags &= ~IOCB_WAITQ; list_del_init(&wait->entry); - init_task_work(&req->task_work, io_req_task_submit); + req->task_work.func = io_req_task_submit; percpu_ref_get(&req->ctx->refs); /* submit ref gets dropped, acquire a new one */ @@ -5083,7 +5166,7 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll, list_del_init(&poll->wait.entry); req->result = mask; - init_task_work(&req->task_work, func); + req->task_work.func = func; percpu_ref_get(&req->ctx->refs); /* @@ -8086,6 +8169,10 @@ static int io_uring_alloc_task_context(struct task_struct *task) io_init_identity(&tctx->__identity); tctx->identity = &tctx->__identity; task->io_uring = tctx; + spin_lock_init(&tctx->task_lock); + INIT_WQ_LIST(&tctx->task_list); + tctx->task_state = 0; + init_task_work(&tctx->task_work, tctx_task_work); return 0; } diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index 35b2d845704d..2eb6d19de336 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -22,6 +22,15 @@ struct io_identity { refcount_t count; }; +struct io_wq_work_node { + struct io_wq_work_node *next; +}; + +struct io_wq_work_list { + struct io_wq_work_node *first; + struct io_wq_work_node *last; +}; + struct io_uring_task { /* submission side */ struct xarray xa; @@ -32,6 +41,11 @@ struct io_uring_task { struct io_identity *identity; atomic_t in_idle; bool sqpoll; + + spinlock_t task_lock; + struct io_wq_work_list task_list; + unsigned long task_state; + struct callback_head task_work; }; #if defined(CONFIG_IO_URING) -- cgit v1.2.3 From a805a4fa4fa376bbc145762bb8b09caa2fa8af48 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Thu, 28 Jan 2021 13:47:30 +0900 Subject: block: introduce zone_write_granularity limit Per ZBC and ZAC specifications, host-managed SMR hard-disks mandate that all writes into sequential write required zones be aligned to the device physical block size. However, NVMe ZNS does not have this constraint and allows write operations into sequential zones to be aligned to the device logical block size. This inconsistency does not help with software portability across device types. To solve this, introduce the zone_write_granularity queue limit to indicate the alignment constraint, in bytes, of write operations into zones of a zoned block device. This new limit is exported as a read-only sysfs queue attribute and the helper blk_queue_zone_write_granularity() introduced for drivers to set this limit. The function blk_queue_set_zoned() is modified to set this new limit to the device logical block size by default. NVMe ZNS devices as well as zoned nullb devices use this default value as is. The scsi disk driver is modified to execute the blk_queue_zone_write_granularity() helper to set the zone write granularity of host-managed SMR disks to the disk physical block size. The accessor functions queue_zone_write_granularity() and bdev_zone_write_granularity() are also introduced. Signed-off-by: Damien Le Moal Reviewed-by: Christoph Hellwig Reviewed-by: Martin K. Petersen Reviewed-by: Chaitanya Kulkarni Reviewed-by: Johannes Thumshirn Signed-off-by: Jens Axboe --- Documentation/block/queue-sysfs.rst | 7 +++++++ block/blk-settings.c | 37 ++++++++++++++++++++++++++++++++++++- block/blk-sysfs.c | 8 ++++++++ drivers/scsi/sd_zbc.c | 8 ++++++++ include/linux/blkdev.h | 15 +++++++++++++++ 5 files changed, 74 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/Documentation/block/queue-sysfs.rst b/Documentation/block/queue-sysfs.rst index edc6e6960b96..4dc7f0d499a8 100644 --- a/Documentation/block/queue-sysfs.rst +++ b/Documentation/block/queue-sysfs.rst @@ -279,4 +279,11 @@ devices are described in the ZBC (Zoned Block Commands) and ZAC do not support zone commands, they will be treated as regular block devices and zoned will report "none". +zone_write_granularity (RO) +--------------------------- +This indicates the alignment constraint, in bytes, for write operations in +sequential zones of zoned block devices (devices with a zoned attributed +that reports "host-managed" or "host-aware"). This value is always 0 for +regular block devices. + Jens Axboe , February 2009 diff --git a/block/blk-settings.c b/block/blk-settings.c index 4c974340f1a9..a1e66165adcf 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -60,6 +60,7 @@ void blk_set_default_limits(struct queue_limits *lim) lim->io_opt = 0; lim->misaligned = 0; lim->zoned = BLK_ZONED_NONE; + lim->zone_write_granularity = 0; } EXPORT_SYMBOL(blk_set_default_limits); @@ -366,6 +367,28 @@ void blk_queue_physical_block_size(struct request_queue *q, unsigned int size) } EXPORT_SYMBOL(blk_queue_physical_block_size); +/** + * blk_queue_zone_write_granularity - set zone write granularity for the queue + * @q: the request queue for the zoned device + * @size: the zone write granularity size, in bytes + * + * Description: + * This should be set to the lowest possible size allowing to write in + * sequential zones of a zoned block device. + */ +void blk_queue_zone_write_granularity(struct request_queue *q, + unsigned int size) +{ + if (WARN_ON_ONCE(!blk_queue_is_zoned(q))) + return; + + q->limits.zone_write_granularity = size; + + if (q->limits.zone_write_granularity < q->limits.logical_block_size) + q->limits.zone_write_granularity = q->limits.logical_block_size; +} +EXPORT_SYMBOL_GPL(blk_queue_zone_write_granularity); + /** * blk_queue_alignment_offset - set physical block alignment offset * @q: the request queue for the device @@ -631,6 +654,8 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, t->discard_granularity; } + t->zone_write_granularity = max(t->zone_write_granularity, + b->zone_write_granularity); t->zoned = max(t->zoned, b->zoned); return ret; } @@ -847,6 +872,8 @@ EXPORT_SYMBOL_GPL(blk_queue_can_use_dma_map_merging); */ void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model) { + struct request_queue *q = disk->queue; + switch (model) { case BLK_ZONED_HM: /* @@ -875,7 +902,15 @@ void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model) break; } - disk->queue->limits.zoned = model; + q->limits.zoned = model; + if (model != BLK_ZONED_NONE) { + /* + * Set the zone write granularity to the device logical block + * size by default. The driver can change this value if needed. + */ + blk_queue_zone_write_granularity(q, + queue_logical_block_size(q)); + } } EXPORT_SYMBOL_GPL(blk_queue_set_zoned); diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index b513f1683af0..ae39c7f3d83d 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -219,6 +219,12 @@ static ssize_t queue_write_zeroes_max_show(struct request_queue *q, char *page) (unsigned long long)q->limits.max_write_zeroes_sectors << 9); } +static ssize_t queue_zone_write_granularity_show(struct request_queue *q, + char *page) +{ + return queue_var_show(queue_zone_write_granularity(q), page); +} + static ssize_t queue_zone_append_max_show(struct request_queue *q, char *page) { unsigned long long max_sectors = q->limits.max_zone_append_sectors; @@ -585,6 +591,7 @@ QUEUE_RO_ENTRY(queue_discard_zeroes_data, "discard_zeroes_data"); QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes"); QUEUE_RO_ENTRY(queue_write_zeroes_max, "write_zeroes_max_bytes"); QUEUE_RO_ENTRY(queue_zone_append_max, "zone_append_max_bytes"); +QUEUE_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity"); QUEUE_RO_ENTRY(queue_zoned, "zoned"); QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones"); @@ -639,6 +646,7 @@ static struct attribute *queue_attrs[] = { &queue_write_same_max_entry.attr, &queue_write_zeroes_max_entry.attr, &queue_zone_append_max_entry.attr, + &queue_zone_write_granularity_entry.attr, &queue_nonrot_entry.attr, &queue_zoned_entry.attr, &queue_nr_zones_entry.attr, diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index cf07b7f93579..8293b29584b3 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -789,6 +789,14 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, unsigned char *buf) blk_queue_max_active_zones(q, 0); nr_zones = round_up(sdkp->capacity, zone_blocks) >> ilog2(zone_blocks); + /* + * Per ZBC and ZAC specifications, writes in sequential write required + * zones of host-managed devices must be aligned to the device physical + * block size. + */ + if (blk_queue_zoned_model(q) == BLK_ZONED_HM) + blk_queue_zone_write_granularity(q, sdkp->physical_block_size); + /* READ16/WRITE16 is mandatory for ZBC disks */ sdkp->device->use_16_for_rw = 1; sdkp->device->use_10_for_rw = 0; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 0dea268bd61b..9149f4a5adb3 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -337,6 +337,7 @@ struct queue_limits { unsigned int max_zone_append_sectors; unsigned int discard_granularity; unsigned int discard_alignment; + unsigned int zone_write_granularity; unsigned short max_segments; unsigned short max_integrity_segments; @@ -1160,6 +1161,8 @@ extern void blk_queue_logical_block_size(struct request_queue *, unsigned int); extern void blk_queue_max_zone_append_sectors(struct request_queue *q, unsigned int max_zone_append_sectors); extern void blk_queue_physical_block_size(struct request_queue *, unsigned int); +void blk_queue_zone_write_granularity(struct request_queue *q, + unsigned int size); extern void blk_queue_alignment_offset(struct request_queue *q, unsigned int alignment); void blk_queue_update_readahead(struct request_queue *q); @@ -1473,6 +1476,18 @@ static inline int bdev_io_opt(struct block_device *bdev) return queue_io_opt(bdev_get_queue(bdev)); } +static inline unsigned int +queue_zone_write_granularity(const struct request_queue *q) +{ + return q->limits.zone_write_granularity; +} + +static inline unsigned int +bdev_zone_write_granularity(struct block_device *bdev) +{ + return queue_zone_write_granularity(bdev_get_queue(bdev)); +} + static inline int queue_alignment_offset(const struct request_queue *q) { if (q->limits.misaligned) -- cgit v1.2.3 From 83fba8c8114748a18e20391565cfdfdf8466075c Mon Sep 17 00:00:00 2001 From: Chao Leng Date: Mon, 1 Feb 2021 11:49:38 +0800 Subject: blk-mq: introduce blk_mq_set_request_complete nvme drivers need to set the state of request to MQ_RQ_COMPLETE when directly complete request in queue_rq. So add blk_mq_set_request_complete. Signed-off-by: Chao Leng Signed-off-by: Christoph Hellwig --- include/linux/blk-mq.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/linux') diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index aabbf6830ffc..2c473c9b8990 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -490,6 +490,18 @@ static inline int blk_mq_request_completed(struct request *rq) return blk_mq_rq_state(rq) == MQ_RQ_COMPLETE; } +/* + * + * Set the state to complete when completing a request from inside ->queue_rq. + * This is used by drivers that want to ensure special complete actions that + * need access to the request are called on failure, e.g. by nvme for + * multipathing. + */ +static inline void blk_mq_set_request_complete(struct request *rq) +{ + WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); +} + void blk_mq_start_request(struct request *rq); void blk_mq_end_request(struct request *rq, blk_status_t error); void __blk_mq_end_request(struct request *rq, blk_status_t error); -- cgit v1.2.3 From 01f810ace9ed37255f27608a0864abebccf0aab3 Mon Sep 17 00:00:00 2001 From: Andrei Matei Date: Sat, 6 Feb 2021 20:10:24 -0500 Subject: bpf: Allow variable-offset stack access Before this patch, variable offset access to the stack was dissalowed for regular instructions, but was allowed for "indirect" accesses (i.e. helpers). This patch removes the restriction, allowing reading and writing to the stack through stack pointers with variable offsets. This makes stack-allocated buffers more usable in programs, and brings stack pointers closer to other types of pointers. The motivation is being able to use stack-allocated buffers for data manipulation. When the stack size limit is sufficient, allocating buffers on the stack is simpler than per-cpu arrays, or other alternatives. In unpriviledged programs, variable-offset reads and writes are disallowed (they were already disallowed for the indirect access case) because the speculative execution checking code doesn't support them. Additionally, when writing through a variable-offset stack pointer, if any pointers are in the accessible range, there's possilibities of later leaking pointers because the write cannot be tracked precisely. Writes with variable offset mark the whole range as initialized, even though we don't know which stack slots are actually written. This is in order to not reject future reads to these slots. Note that this doesn't affect writes done through helpers; like before, helpers need the whole stack range to be initialized to begin with. All the stack slots are in range are considered scalars after the write; variable-offset register spills are not tracked. For reads, all the stack slots in the variable range needs to be initialized (but see above about what writes do), otherwise the read is rejected. All register spilled in stack slots that might be read are marked as having been read, however reads through such pointers don't do register filling; the target register will always be either a scalar or a constant zero. Signed-off-by: Andrei Matei Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20210207011027.676572-2-andreimatei1@gmail.com --- include/linux/bpf.h | 5 + include/linux/bpf_verifier.h | 3 +- kernel/bpf/verifier.c | 657 +++++++++++++++++++++++++++++++++---------- 3 files changed, 518 insertions(+), 147 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 321966fc35db..079162bbd387 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1290,6 +1290,11 @@ static inline bool bpf_allow_ptr_leaks(void) return perfmon_capable(); } +static inline bool bpf_allow_uninit_stack(void) +{ + return perfmon_capable(); +} + static inline bool bpf_allow_ptr_to_map_access(void) { return perfmon_capable(); diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index dfe6f85d97dd..532c97836d0d 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -195,7 +195,7 @@ struct bpf_func_state { * 0 = main function, 1 = first callee. */ u32 frameno; - /* subprog number == index within subprog_stack_depth + /* subprog number == index within subprog_info * zero == main subprog */ u32 subprogno; @@ -404,6 +404,7 @@ struct bpf_verifier_env { u32 used_btf_cnt; /* number of used BTF objects */ u32 id_gen; /* used to generate unique reg IDs */ bool allow_ptr_leaks; + bool allow_uninit_stack; bool allow_ptr_to_map_access; bool bpf_capable; bool bypass_spec_v1; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 15694246f854..400d79e99fc8 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2275,12 +2275,14 @@ static void save_register_state(struct bpf_func_state *state, state->stack[spi].slot_type[i] = STACK_SPILL; } -/* check_stack_read/write functions track spill/fill of registers, +/* check_stack_{read,write}_fixed_off functions track spill/fill of registers, * stack boundary and alignment are checked in check_mem_access() */ -static int check_stack_write(struct bpf_verifier_env *env, - struct bpf_func_state *state, /* func where register points to */ - int off, int size, int value_regno, int insn_idx) +static int check_stack_write_fixed_off(struct bpf_verifier_env *env, + /* stack frame we're writing to */ + struct bpf_func_state *state, + int off, int size, int value_regno, + int insn_idx) { struct bpf_func_state *cur; /* state of the current function */ int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; @@ -2406,9 +2408,175 @@ static int check_stack_write(struct bpf_verifier_env *env, return 0; } -static int check_stack_read(struct bpf_verifier_env *env, - struct bpf_func_state *reg_state /* func where register points to */, - int off, int size, int value_regno) +/* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is + * known to contain a variable offset. + * This function checks whether the write is permitted and conservatively + * tracks the effects of the write, considering that each stack slot in the + * dynamic range is potentially written to. + * + * 'off' includes 'regno->off'. + * 'value_regno' can be -1, meaning that an unknown value is being written to + * the stack. + * + * Spilled pointers in range are not marked as written because we don't know + * what's going to be actually written. This means that read propagation for + * future reads cannot be terminated by this write. + * + * For privileged programs, uninitialized stack slots are considered + * initialized by this write (even though we don't know exactly what offsets + * are going to be written to). The idea is that we don't want the verifier to + * reject future reads that access slots written to through variable offsets. + */ +static int check_stack_write_var_off(struct bpf_verifier_env *env, + /* func where register points to */ + struct bpf_func_state *state, + int ptr_regno, int off, int size, + int value_regno, int insn_idx) +{ + struct bpf_func_state *cur; /* state of the current function */ + int min_off, max_off; + int i, err; + struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL; + bool writing_zero = false; + /* set if the fact that we're writing a zero is used to let any + * stack slots remain STACK_ZERO + */ + bool zero_used = false; + + cur = env->cur_state->frame[env->cur_state->curframe]; + ptr_reg = &cur->regs[ptr_regno]; + min_off = ptr_reg->smin_value + off; + max_off = ptr_reg->smax_value + off + size; + if (value_regno >= 0) + value_reg = &cur->regs[value_regno]; + if (value_reg && register_is_null(value_reg)) + writing_zero = true; + + err = realloc_func_state(state, round_up(-min_off, BPF_REG_SIZE), + state->acquired_refs, true); + if (err) + return err; + + + /* Variable offset writes destroy any spilled pointers in range. */ + for (i = min_off; i < max_off; i++) { + u8 new_type, *stype; + int slot, spi; + + slot = -i - 1; + spi = slot / BPF_REG_SIZE; + stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; + + if (!env->allow_ptr_leaks + && *stype != NOT_INIT + && *stype != SCALAR_VALUE) { + /* Reject the write if there's are spilled pointers in + * range. If we didn't reject here, the ptr status + * would be erased below (even though not all slots are + * actually overwritten), possibly opening the door to + * leaks. + */ + verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", + insn_idx, i); + return -EINVAL; + } + + /* Erase all spilled pointers. */ + state->stack[spi].spilled_ptr.type = NOT_INIT; + + /* Update the slot type. */ + new_type = STACK_MISC; + if (writing_zero && *stype == STACK_ZERO) { + new_type = STACK_ZERO; + zero_used = true; + } + /* If the slot is STACK_INVALID, we check whether it's OK to + * pretend that it will be initialized by this write. The slot + * might not actually be written to, and so if we mark it as + * initialized future reads might leak uninitialized memory. + * For privileged programs, we will accept such reads to slots + * that may or may not be written because, if we're reject + * them, the error would be too confusing. + */ + if (*stype == STACK_INVALID && !env->allow_uninit_stack) { + verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", + insn_idx, i); + return -EINVAL; + } + *stype = new_type; + } + if (zero_used) { + /* backtracking doesn't work for STACK_ZERO yet. */ + err = mark_chain_precision(env, value_regno); + if (err) + return err; + } + return 0; +} + +/* When register 'dst_regno' is assigned some values from stack[min_off, + * max_off), we set the register's type according to the types of the + * respective stack slots. If all the stack values are known to be zeros, then + * so is the destination reg. Otherwise, the register is considered to be + * SCALAR. This function does not deal with register filling; the caller must + * ensure that all spilled registers in the stack range have been marked as + * read. + */ +static void mark_reg_stack_read(struct bpf_verifier_env *env, + /* func where src register points to */ + struct bpf_func_state *ptr_state, + int min_off, int max_off, int dst_regno) +{ + struct bpf_verifier_state *vstate = env->cur_state; + struct bpf_func_state *state = vstate->frame[vstate->curframe]; + int i, slot, spi; + u8 *stype; + int zeros = 0; + + for (i = min_off; i < max_off; i++) { + slot = -i - 1; + spi = slot / BPF_REG_SIZE; + stype = ptr_state->stack[spi].slot_type; + if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) + break; + zeros++; + } + if (zeros == max_off - min_off) { + /* any access_size read into register is zero extended, + * so the whole register == const_zero + */ + __mark_reg_const_zero(&state->regs[dst_regno]); + /* backtracking doesn't support STACK_ZERO yet, + * so mark it precise here, so that later + * backtracking can stop here. + * Backtracking may not need this if this register + * doesn't participate in pointer adjustment. + * Forward propagation of precise flag is not + * necessary either. This mark is only to stop + * backtracking. Any register that contributed + * to const 0 was marked precise before spill. + */ + state->regs[dst_regno].precise = true; + } else { + /* have read misc data from the stack */ + mark_reg_unknown(env, state->regs, dst_regno); + } + state->regs[dst_regno].live |= REG_LIVE_WRITTEN; +} + +/* Read the stack at 'off' and put the results into the register indicated by + * 'dst_regno'. It handles reg filling if the addressed stack slot is a + * spilled reg. + * + * 'dst_regno' can be -1, meaning that the read value is not going to a + * register. + * + * The access is assumed to be within the current stack bounds. + */ +static int check_stack_read_fixed_off(struct bpf_verifier_env *env, + /* func where src register points to */ + struct bpf_func_state *reg_state, + int off, int size, int dst_regno) { struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; @@ -2416,11 +2584,6 @@ static int check_stack_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg; u8 *stype; - if (reg_state->allocated_stack <= slot) { - verbose(env, "invalid read from stack off %d+0 size %d\n", - off, size); - return -EACCES; - } stype = reg_state->stack[spi].slot_type; reg = ®_state->stack[spi].spilled_ptr; @@ -2431,9 +2594,9 @@ static int check_stack_read(struct bpf_verifier_env *env, verbose(env, "invalid size of register fill\n"); return -EACCES; } - if (value_regno >= 0) { - mark_reg_unknown(env, state->regs, value_regno); - state->regs[value_regno].live |= REG_LIVE_WRITTEN; + if (dst_regno >= 0) { + mark_reg_unknown(env, state->regs, dst_regno); + state->regs[dst_regno].live |= REG_LIVE_WRITTEN; } mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); return 0; @@ -2445,16 +2608,16 @@ static int check_stack_read(struct bpf_verifier_env *env, } } - if (value_regno >= 0) { + if (dst_regno >= 0) { /* restore register state from stack */ - state->regs[value_regno] = *reg; + state->regs[dst_regno] = *reg; /* mark reg as written since spilled pointer state likely * has its liveness marks cleared by is_state_visited() * which resets stack/reg liveness for state transitions */ - state->regs[value_regno].live |= REG_LIVE_WRITTEN; + state->regs[dst_regno].live |= REG_LIVE_WRITTEN; } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { - /* If value_regno==-1, the caller is asking us whether + /* If dst_regno==-1, the caller is asking us whether * it is acceptable to use this value as a SCALAR_VALUE * (e.g. for XADD). * We must not allow unprivileged callers to do that @@ -2466,70 +2629,167 @@ static int check_stack_read(struct bpf_verifier_env *env, } mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); } else { - int zeros = 0; + u8 type; for (i = 0; i < size; i++) { - if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC) + type = stype[(slot - i) % BPF_REG_SIZE]; + if (type == STACK_MISC) continue; - if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) { - zeros++; + if (type == STACK_ZERO) continue; - } verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); return -EACCES; } mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); - if (value_regno >= 0) { - if (zeros == size) { - /* any size read into register is zero extended, - * so the whole register == const_zero - */ - __mark_reg_const_zero(&state->regs[value_regno]); - /* backtracking doesn't support STACK_ZERO yet, - * so mark it precise here, so that later - * backtracking can stop here. - * Backtracking may not need this if this register - * doesn't participate in pointer adjustment. - * Forward propagation of precise flag is not - * necessary either. This mark is only to stop - * backtracking. Any register that contributed - * to const 0 was marked precise before spill. - */ - state->regs[value_regno].precise = true; - } else { - /* have read misc data from the stack */ - mark_reg_unknown(env, state->regs, value_regno); - } - state->regs[value_regno].live |= REG_LIVE_WRITTEN; - } + if (dst_regno >= 0) + mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); } return 0; } -static int check_stack_access(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, - int off, int size) +enum stack_access_src { + ACCESS_DIRECT = 1, /* the access is performed by an instruction */ + ACCESS_HELPER = 2, /* the access is performed by a helper */ +}; + +static int check_stack_range_initialized(struct bpf_verifier_env *env, + int regno, int off, int access_size, + bool zero_size_allowed, + enum stack_access_src type, + struct bpf_call_arg_meta *meta); + +static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) +{ + return cur_regs(env) + regno; +} + +/* Read the stack at 'ptr_regno + off' and put the result into the register + * 'dst_regno'. + * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'), + * but not its variable offset. + * 'size' is assumed to be <= reg size and the access is assumed to be aligned. + * + * As opposed to check_stack_read_fixed_off, this function doesn't deal with + * filling registers (i.e. reads of spilled register cannot be detected when + * the offset is not fixed). We conservatively mark 'dst_regno' as containing + * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable + * offset; for a fixed offset check_stack_read_fixed_off should be used + * instead. + */ +static int check_stack_read_var_off(struct bpf_verifier_env *env, + int ptr_regno, int off, int size, int dst_regno) { - /* Stack accesses must be at a fixed offset, so that we - * can determine what type of data were returned. See - * check_stack_read(). + /* The state of the source register. */ + struct bpf_reg_state *reg = reg_state(env, ptr_regno); + struct bpf_func_state *ptr_state = func(env, reg); + int err; + int min_off, max_off; + + /* Note that we pass a NULL meta, so raw access will not be permitted. */ - if (!tnum_is_const(reg->var_off)) { + err = check_stack_range_initialized(env, ptr_regno, off, size, + false, ACCESS_DIRECT, NULL); + if (err) + return err; + + min_off = reg->smin_value + off; + max_off = reg->smax_value + off; + mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); + return 0; +} + +/* check_stack_read dispatches to check_stack_read_fixed_off or + * check_stack_read_var_off. + * + * The caller must ensure that the offset falls within the allocated stack + * bounds. + * + * 'dst_regno' is a register which will receive the value from the stack. It + * can be -1, meaning that the read value is not going to a register. + */ +static int check_stack_read(struct bpf_verifier_env *env, + int ptr_regno, int off, int size, + int dst_regno) +{ + struct bpf_reg_state *reg = reg_state(env, ptr_regno); + struct bpf_func_state *state = func(env, reg); + int err; + /* Some accesses are only permitted with a static offset. */ + bool var_off = !tnum_is_const(reg->var_off); + + /* The offset is required to be static when reads don't go to a + * register, in order to not leak pointers (see + * check_stack_read_fixed_off). + */ + if (dst_regno < 0 && var_off) { char tn_buf[48]; tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); - verbose(env, "variable stack access var_off=%s off=%d size=%d\n", + verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", tn_buf, off, size); return -EACCES; } + /* Variable offset is prohibited for unprivileged mode for simplicity + * since it requires corresponding support in Spectre masking for stack + * ALU. See also retrieve_ptr_limit(). + */ + if (!env->bypass_spec_v1 && var_off) { + char tn_buf[48]; - if (off >= 0 || off < -MAX_BPF_STACK) { - verbose(env, "invalid stack off=%d size=%d\n", off, size); + tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", + ptr_regno, tn_buf); return -EACCES; } - return 0; + if (!var_off) { + off += reg->var_off.value; + err = check_stack_read_fixed_off(env, state, off, size, + dst_regno); + } else { + /* Variable offset stack reads need more conservative handling + * than fixed offset ones. Note that dst_regno >= 0 on this + * branch. + */ + err = check_stack_read_var_off(env, ptr_regno, off, size, + dst_regno); + } + return err; +} + + +/* check_stack_write dispatches to check_stack_write_fixed_off or + * check_stack_write_var_off. + * + * 'ptr_regno' is the register used as a pointer into the stack. + * 'off' includes 'ptr_regno->off', but not its variable offset (if any). + * 'value_regno' is the register whose value we're writing to the stack. It can + * be -1, meaning that we're not writing from a register. + * + * The caller must ensure that the offset falls within the maximum stack size. + */ +static int check_stack_write(struct bpf_verifier_env *env, + int ptr_regno, int off, int size, + int value_regno, int insn_idx) +{ + struct bpf_reg_state *reg = reg_state(env, ptr_regno); + struct bpf_func_state *state = func(env, reg); + int err; + + if (tnum_is_const(reg->var_off)) { + off += reg->var_off.value; + err = check_stack_write_fixed_off(env, state, off, size, + value_regno, insn_idx); + } else { + /* Variable offset stack reads need more conservative handling + * than fixed offset ones. + */ + err = check_stack_write_var_off(env, state, + ptr_regno, off, size, + value_regno, insn_idx); + } + return err; } static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, @@ -2862,11 +3122,6 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, return -EACCES; } -static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) -{ - return cur_regs(env) + regno; -} - static bool is_pointer_value(struct bpf_verifier_env *env, int regno) { return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); @@ -2985,8 +3240,8 @@ static int check_ptr_alignment(struct bpf_verifier_env *env, break; case PTR_TO_STACK: pointer_desc = "stack "; - /* The stack spill tracking logic in check_stack_write() - * and check_stack_read() relies on stack accesses being + /* The stack spill tracking logic in check_stack_write_fixed_off() + * and check_stack_read_fixed_off() relies on stack accesses being * aligned. */ strict = true; @@ -3402,6 +3657,91 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env, return 0; } +/* Check that the stack access at the given offset is within bounds. The + * maximum valid offset is -1. + * + * The minimum valid offset is -MAX_BPF_STACK for writes, and + * -state->allocated_stack for reads. + */ +static int check_stack_slot_within_bounds(int off, + struct bpf_func_state *state, + enum bpf_access_type t) +{ + int min_valid_off; + + if (t == BPF_WRITE) + min_valid_off = -MAX_BPF_STACK; + else + min_valid_off = -state->allocated_stack; + + if (off < min_valid_off || off > -1) + return -EACCES; + return 0; +} + +/* Check that the stack access at 'regno + off' falls within the maximum stack + * bounds. + * + * 'off' includes `regno->offset`, but not its dynamic part (if any). + */ +static int check_stack_access_within_bounds( + struct bpf_verifier_env *env, + int regno, int off, int access_size, + enum stack_access_src src, enum bpf_access_type type) +{ + struct bpf_reg_state *regs = cur_regs(env); + struct bpf_reg_state *reg = regs + regno; + struct bpf_func_state *state = func(env, reg); + int min_off, max_off; + int err; + char *err_extra; + + if (src == ACCESS_HELPER) + /* We don't know if helpers are reading or writing (or both). */ + err_extra = " indirect access to"; + else if (type == BPF_READ) + err_extra = " read from"; + else + err_extra = " write to"; + + if (tnum_is_const(reg->var_off)) { + min_off = reg->var_off.value + off; + if (access_size > 0) + max_off = min_off + access_size - 1; + else + max_off = min_off; + } else { + if (reg->smax_value >= BPF_MAX_VAR_OFF || + reg->smin_value <= -BPF_MAX_VAR_OFF) { + verbose(env, "invalid unbounded variable-offset%s stack R%d\n", + err_extra, regno); + return -EACCES; + } + min_off = reg->smin_value + off; + if (access_size > 0) + max_off = reg->smax_value + off + access_size - 1; + else + max_off = min_off; + } + + err = check_stack_slot_within_bounds(min_off, state, type); + if (!err) + err = check_stack_slot_within_bounds(max_off, state, type); + + if (err) { + if (tnum_is_const(reg->var_off)) { + verbose(env, "invalid%s stack R%d off=%d size=%d\n", + err_extra, regno, off, access_size); + } else { + char tn_buf[48]; + + tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", + err_extra, regno, tn_buf, access_size); + } + } + return err; +} /* check whether memory at (regno + off) is accessible for t = (read | write) * if t==write, value_regno is a register which value is stored into memory @@ -3517,8 +3857,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn } } else if (reg->type == PTR_TO_STACK) { - off += reg->var_off.value; - err = check_stack_access(env, reg, off, size); + /* Basic bounds checks. */ + err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); if (err) return err; @@ -3527,12 +3867,12 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn if (err) return err; - if (t == BPF_WRITE) - err = check_stack_write(env, state, off, size, - value_regno, insn_idx); - else - err = check_stack_read(env, state, off, size, + if (t == BPF_READ) + err = check_stack_read(env, regno, off, size, value_regno); + else + err = check_stack_write(env, regno, off, size, + value_regno, insn_idx); } else if (reg_is_pkt_pointer(reg)) { if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { verbose(env, "cannot write into packet\n"); @@ -3699,49 +4039,53 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i return 0; } -static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno, - int off, int access_size, - bool zero_size_allowed) +/* When register 'regno' is used to read the stack (either directly or through + * a helper function) make sure that it's within stack boundary and, depending + * on the access type, that all elements of the stack are initialized. + * + * 'off' includes 'regno->off', but not its dynamic part (if any). + * + * All registers that have been spilled on the stack in the slots within the + * read offsets are marked as read. + */ +static int check_stack_range_initialized( + struct bpf_verifier_env *env, int regno, int off, + int access_size, bool zero_size_allowed, + enum stack_access_src type, struct bpf_call_arg_meta *meta) { struct bpf_reg_state *reg = reg_state(env, regno); + struct bpf_func_state *state = func(env, reg); + int err, min_off, max_off, i, j, slot, spi; + char *err_extra = type == ACCESS_HELPER ? " indirect" : ""; + enum bpf_access_type bounds_check_type; + /* Some accesses can write anything into the stack, others are + * read-only. + */ + bool clobber = false; - if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 || - access_size < 0 || (access_size == 0 && !zero_size_allowed)) { - if (tnum_is_const(reg->var_off)) { - verbose(env, "invalid stack type R%d off=%d access_size=%d\n", - regno, off, access_size); - } else { - char tn_buf[48]; - - tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); - verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n", - regno, tn_buf, access_size); - } + if (access_size == 0 && !zero_size_allowed) { + verbose(env, "invalid zero-sized read\n"); return -EACCES; } - return 0; -} -/* when register 'regno' is passed into function that will read 'access_size' - * bytes from that pointer, make sure that it's within stack boundary - * and all elements of stack are initialized. - * Unlike most pointer bounds-checking functions, this one doesn't take an - * 'off' argument, so it has to add in reg->off itself. - */ -static int check_stack_boundary(struct bpf_verifier_env *env, int regno, - int access_size, bool zero_size_allowed, - struct bpf_call_arg_meta *meta) -{ - struct bpf_reg_state *reg = reg_state(env, regno); - struct bpf_func_state *state = func(env, reg); - int err, min_off, max_off, i, j, slot, spi; + if (type == ACCESS_HELPER) { + /* The bounds checks for writes are more permissive than for + * reads. However, if raw_mode is not set, we'll do extra + * checks below. + */ + bounds_check_type = BPF_WRITE; + clobber = true; + } else { + bounds_check_type = BPF_READ; + } + err = check_stack_access_within_bounds(env, regno, off, access_size, + type, bounds_check_type); + if (err) + return err; + if (tnum_is_const(reg->var_off)) { - min_off = max_off = reg->var_off.value + reg->off; - err = __check_stack_boundary(env, regno, min_off, access_size, - zero_size_allowed); - if (err) - return err; + min_off = max_off = reg->var_off.value + off; } else { /* Variable offset is prohibited for unprivileged mode for * simplicity since it requires corresponding support in @@ -3752,8 +4096,8 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, char tn_buf[48]; tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); - verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n", - regno, tn_buf); + verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", + regno, err_extra, tn_buf); return -EACCES; } /* Only initialized buffer on stack is allowed to be accessed @@ -3765,28 +4109,8 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, if (meta && meta->raw_mode) meta = NULL; - if (reg->smax_value >= BPF_MAX_VAR_OFF || - reg->smax_value <= -BPF_MAX_VAR_OFF) { - verbose(env, "R%d unbounded indirect variable offset stack access\n", - regno); - return -EACCES; - } - min_off = reg->smin_value + reg->off; - max_off = reg->smax_value + reg->off; - err = __check_stack_boundary(env, regno, min_off, access_size, - zero_size_allowed); - if (err) { - verbose(env, "R%d min value is outside of stack bound\n", - regno); - return err; - } - err = __check_stack_boundary(env, regno, max_off, access_size, - zero_size_allowed); - if (err) { - verbose(env, "R%d max value is outside of stack bound\n", - regno); - return err; - } + min_off = reg->smin_value + off; + max_off = reg->smax_value + off; } if (meta && meta->raw_mode) { @@ -3806,8 +4130,10 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, if (*stype == STACK_MISC) goto mark; if (*stype == STACK_ZERO) { - /* helper can write anything into the stack */ - *stype = STACK_MISC; + if (clobber) { + /* helper can write anything into the stack */ + *stype = STACK_MISC; + } goto mark; } @@ -3818,22 +4144,24 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, if (state->stack[spi].slot_type[0] == STACK_SPILL && (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || env->allow_ptr_leaks)) { - __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); - for (j = 0; j < BPF_REG_SIZE; j++) - state->stack[spi].slot_type[j] = STACK_MISC; + if (clobber) { + __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); + for (j = 0; j < BPF_REG_SIZE; j++) + state->stack[spi].slot_type[j] = STACK_MISC; + } goto mark; } err: if (tnum_is_const(reg->var_off)) { - verbose(env, "invalid indirect read from stack off %d+%d size %d\n", - min_off, i - min_off, access_size); + verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", + err_extra, regno, min_off, i - min_off, access_size); } else { char tn_buf[48]; tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); - verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n", - tn_buf, i - min_off, access_size); + verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", + err_extra, regno, tn_buf, i - min_off, access_size); } return -EACCES; mark: @@ -3882,8 +4210,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, "rdwr", &env->prog->aux->max_rdwr_access); case PTR_TO_STACK: - return check_stack_boundary(env, regno, access_size, - zero_size_allowed, meta); + return check_stack_range_initialized( + env, + regno, reg->off, access_size, + zero_size_allowed, ACCESS_HELPER, meta); default: /* scalar_value or invalid ptr */ /* Allow zero-byte read from NULL, regardless of pointer type */ if (zero_size_allowed && access_size == 0 && @@ -5547,6 +5877,41 @@ do_sim: return !ret ? -EFAULT : 0; } +/* check that stack access falls within stack limits and that 'reg' doesn't + * have a variable offset. + * + * Variable offset is prohibited for unprivileged mode for simplicity since it + * requires corresponding support in Spectre masking for stack ALU. See also + * retrieve_ptr_limit(). + * + * + * 'off' includes 'reg->off'. + */ +static int check_stack_access_for_ptr_arithmetic( + struct bpf_verifier_env *env, + int regno, + const struct bpf_reg_state *reg, + int off) +{ + if (!tnum_is_const(reg->var_off)) { + char tn_buf[48]; + + tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); + verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", + regno, tn_buf, off); + return -EACCES; + } + + if (off >= 0 || off < -MAX_BPF_STACK) { + verbose(env, "R%d stack pointer arithmetic goes out of range, " + "prohibited for !root; off=%d\n", regno, off); + return -EACCES; + } + + return 0; +} + + /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. * Caller should also handle BPF_MOV case separately. * If we return -EACCES, caller may want to try again treating pointer as a @@ -5790,10 +6155,9 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, "prohibited for !root\n", dst); return -EACCES; } else if (dst_reg->type == PTR_TO_STACK && - check_stack_access(env, dst_reg, dst_reg->off + - dst_reg->var_off.value, 1)) { - verbose(env, "R%d stack pointer arithmetic goes out of range, " - "prohibited for !root\n", dst); + check_stack_access_for_ptr_arithmetic( + env, dst, dst_reg, dst_reg->off + + dst_reg->var_off.value)) { return -EACCES; } } @@ -12129,6 +12493,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, env->strict_alignment = false; env->allow_ptr_leaks = bpf_allow_ptr_leaks(); + env->allow_uninit_stack = bpf_allow_uninit_stack(); env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access(); env->bypass_spec_v1 = bpf_bypass_spec_v1(); env->bypass_spec_v4 = bpf_bypass_spec_v4(); -- cgit v1.2.3 From 5b74df80f301e872143fa716f3f4361b2e293e19 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 4 Jan 2021 09:38:57 +0200 Subject: net/mlx5: Delete device list leftover Device list is not stored in mlx5_priv anymore, so delete it as it's not used. Signed-off-by: Leon Romanovsky Signed-off-by: Saeed Mahameed --- include/linux/mlx5/driver.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 88197b87bd81..936302b2c141 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -588,7 +588,6 @@ struct mlx5_priv { /* end: alloc staff */ struct dentry *dbg_root; - struct list_head dev_list; struct list_head ctx_list; spinlock_t ctx_lock; struct mlx5_adev **adev; -- cgit v1.2.3 From 21a4e356d3588806307555c149b80cec3dedb180 Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Mon, 1 Feb 2021 15:48:12 +0100 Subject: Drivers: hv: vmbus: Restrict vmbus_devices on isolated guests Only the VSCs or ICs that have been hardened and that are critical for the successful adoption of Confidential VMs should be allowed if the guest is running isolated. This change reduces the footprint of the code that will be exercised by Confidential VMs and hence the exposure to bugs and vulnerabilities. Signed-off-by: Andrea Parri (Microsoft) Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20210201144814.2701-3-parri.andrea@gmail.com Signed-off-by: Wei Liu --- drivers/hv/channel_mgmt.c | 38 ++++++++++++++++++++++++++++++++++++++ include/linux/hyperv.h | 1 + 2 files changed, 39 insertions(+) (limited to 'include/linux') diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 68950a1e4b63..f0ed730e2e4e 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -31,101 +31,118 @@ const struct vmbus_device vmbus_devs[] = { { .dev_type = HV_IDE, HV_IDE_GUID, .perf_device = true, + .allowed_in_isolated = false, }, /* SCSI */ { .dev_type = HV_SCSI, HV_SCSI_GUID, .perf_device = true, + .allowed_in_isolated = true, }, /* Fibre Channel */ { .dev_type = HV_FC, HV_SYNTHFC_GUID, .perf_device = true, + .allowed_in_isolated = false, }, /* Synthetic NIC */ { .dev_type = HV_NIC, HV_NIC_GUID, .perf_device = true, + .allowed_in_isolated = true, }, /* Network Direct */ { .dev_type = HV_ND, HV_ND_GUID, .perf_device = true, + .allowed_in_isolated = false, }, /* PCIE */ { .dev_type = HV_PCIE, HV_PCIE_GUID, .perf_device = false, + .allowed_in_isolated = false, }, /* Synthetic Frame Buffer */ { .dev_type = HV_FB, HV_SYNTHVID_GUID, .perf_device = false, + .allowed_in_isolated = false, }, /* Synthetic Keyboard */ { .dev_type = HV_KBD, HV_KBD_GUID, .perf_device = false, + .allowed_in_isolated = false, }, /* Synthetic MOUSE */ { .dev_type = HV_MOUSE, HV_MOUSE_GUID, .perf_device = false, + .allowed_in_isolated = false, }, /* KVP */ { .dev_type = HV_KVP, HV_KVP_GUID, .perf_device = false, + .allowed_in_isolated = false, }, /* Time Synch */ { .dev_type = HV_TS, HV_TS_GUID, .perf_device = false, + .allowed_in_isolated = true, }, /* Heartbeat */ { .dev_type = HV_HB, HV_HEART_BEAT_GUID, .perf_device = false, + .allowed_in_isolated = true, }, /* Shutdown */ { .dev_type = HV_SHUTDOWN, HV_SHUTDOWN_GUID, .perf_device = false, + .allowed_in_isolated = true, }, /* File copy */ { .dev_type = HV_FCOPY, HV_FCOPY_GUID, .perf_device = false, + .allowed_in_isolated = false, }, /* Backup */ { .dev_type = HV_BACKUP, HV_VSS_GUID, .perf_device = false, + .allowed_in_isolated = false, }, /* Dynamic Memory */ { .dev_type = HV_DM, HV_DM_GUID, .perf_device = false, + .allowed_in_isolated = false, }, /* Unknown GUID */ { .dev_type = HV_UNKNOWN, .perf_device = false, + .allowed_in_isolated = false, }, }; @@ -903,6 +920,20 @@ find_primary_channel_by_offer(const struct vmbus_channel_offer_channel *offer) return channel; } +static bool vmbus_is_valid_device(const guid_t *guid) +{ + u16 i; + + if (!hv_is_isolation_supported()) + return true; + + for (i = 0; i < ARRAY_SIZE(vmbus_devs); i++) { + if (guid_equal(guid, &vmbus_devs[i].guid)) + return vmbus_devs[i].allowed_in_isolated; + } + return false; +} + /* * vmbus_onoffer - Handler for channel offers from vmbus in parent partition. * @@ -917,6 +948,13 @@ static void vmbus_onoffer(struct vmbus_channel_message_header *hdr) trace_vmbus_onoffer(offer); + if (!vmbus_is_valid_device(&offer->offer.if_type)) { + pr_err_ratelimited("Invalid offer %d from the host supporting isolation\n", + offer->child_relid); + atomic_dec(&vmbus_connection.offer_in_progress); + return; + } + oldchannel = find_primary_channel_by_offer(offer); if (oldchannel != NULL) { diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index f0d48a368f13..e3426f8c12db 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -789,6 +789,7 @@ struct vmbus_device { u16 dev_type; guid_t guid; bool perf_device; + bool allowed_in_isolated; }; #define VMBUS_DEFAULT_MAX_PKT_SIZE 4096 -- cgit v1.2.3 From 78785010d428f7755bf51d1c08cb2566a73dc7f5 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 1 Feb 2021 11:43:34 -0600 Subject: hv: hyperv.h: Replace one-element array with flexible-array in struct icmsg_negotiate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is a regular need in the kernel to provide a way to declare having a dynamically sized set of trailing elements in a structure. Kernel code should always use “flexible array members”[1] for these cases. The older style of one-element or zero-length arrays should no longer be used[2]. Refactor the code according to the use of a flexible-array member in struct icmsg_negotiate, instead of a one-element array. Also, this helps the ongoing efforts to enable -Warray-bounds and fix the following warnings: drivers/hv/channel_mgmt.c:315:23: warning: array subscript 1 is above array bounds of ‘struct ic_version[1]’ [-Warray-bounds] drivers/hv/channel_mgmt.c:316:23: warning: array subscript 1 is above array bounds of ‘struct ic_version[1]’ [-Warray-bounds] [1] https://en.wikipedia.org/wiki/Flexible_array_member [2] https://www.kernel.org/doc/html/v5.9/process/deprecated.html#zero-length-and-one-element-arrays Link: https://github.com/KSPP/linux/issues/79 Link: https://github.com/KSPP/linux/issues/109 Signed-off-by: Gustavo A. R. Silva Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20210201174334.GA171933@embeddedor Signed-off-by: Wei Liu --- include/linux/hyperv.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index e3426f8c12db..9dd22af1b7f6 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -1529,14 +1529,14 @@ struct icmsg_hdr { #define IC_VERSION_NEGOTIATION_MAX_VER_COUNT 100 #define ICMSG_HDR (sizeof(struct vmbuspipe_hdr) + sizeof(struct icmsg_hdr)) #define ICMSG_NEGOTIATE_PKT_SIZE(icframe_vercnt, icmsg_vercnt) \ - (ICMSG_HDR + offsetof(struct icmsg_negotiate, icversion_data) + \ + (ICMSG_HDR + sizeof(struct icmsg_negotiate) + \ (((icframe_vercnt) + (icmsg_vercnt)) * sizeof(struct ic_version))) struct icmsg_negotiate { u16 icframe_vercnt; u16 icmsg_vercnt; u32 reserved; - struct ic_version icversion_data[1]; /* any size array */ + struct ic_version icversion_data[]; /* any size array */ } __packed; struct shutdown_msg_data { -- cgit v1.2.3 From 700d4796ef59f5faf240d307839bd419e2b6bdff Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 9 Feb 2021 19:36:26 -0800 Subject: bpf: Optimize program stats Move bpf_prog_stats from prog->aux into prog to avoid one extra load in critical path of program execution. Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20210210033634.62081-2-alexei.starovoitov@gmail.com --- include/linux/bpf.h | 8 -------- include/linux/filter.h | 14 +++++++++++--- kernel/bpf/core.c | 8 ++++---- kernel/bpf/syscall.c | 2 +- kernel/bpf/trampoline.c | 2 +- kernel/bpf/verifier.c | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 079162bbd387..5a388955e6b6 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -507,12 +506,6 @@ enum bpf_cgroup_storage_type { */ #define MAX_BPF_FUNC_ARGS 12 -struct bpf_prog_stats { - u64 cnt; - u64 nsecs; - struct u64_stats_sync syncp; -} __aligned(2 * sizeof(u64)); - struct btf_func_model { u8 ret_size; u8 nr_args; @@ -845,7 +838,6 @@ struct bpf_prog_aux { u32 linfo_idx; u32 num_exentries; struct exception_table_entry *extable; - struct bpf_prog_stats __percpu *stats; union { struct work_struct work; struct rcu_head rcu; diff --git a/include/linux/filter.h b/include/linux/filter.h index 5b3137d7b690..cecb03c9d251 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -539,6 +540,12 @@ struct bpf_binary_header { u8 image[] __aligned(BPF_IMAGE_ALIGNMENT); }; +struct bpf_prog_stats { + u64 cnt; + u64 nsecs; + struct u64_stats_sync syncp; +} __aligned(2 * sizeof(u64)); + struct bpf_prog { u16 pages; /* Number of allocated pages */ u16 jited:1, /* Is our filter JIT'ed? */ @@ -557,10 +564,11 @@ struct bpf_prog { u32 len; /* Number of filter blocks */ u32 jited_len; /* Size of jited insns in bytes */ u8 tag[BPF_TAG_SIZE]; - struct bpf_prog_aux *aux; /* Auxiliary fields */ - struct sock_fprog_kern *orig_prog; /* Original BPF program */ + struct bpf_prog_stats __percpu *stats; unsigned int (*bpf_func)(const void *ctx, const struct bpf_insn *insn); + struct bpf_prog_aux *aux; /* Auxiliary fields */ + struct sock_fprog_kern *orig_prog; /* Original BPF program */ /* Instructions for interpreter */ struct sock_filter insns[0]; struct bpf_insn insnsi[]; @@ -581,7 +589,7 @@ DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key); struct bpf_prog_stats *__stats; \ u64 __start = sched_clock(); \ __ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func); \ - __stats = this_cpu_ptr(prog->aux->stats); \ + __stats = this_cpu_ptr(prog->stats); \ u64_stats_update_begin(&__stats->syncp); \ __stats->cnt++; \ __stats->nsecs += sched_clock() - __start; \ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 5bbd4884ff7a..2cf71fd39c22 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -114,8 +114,8 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) if (!prog) return NULL; - prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); - if (!prog->aux->stats) { + prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); + if (!prog->stats) { kfree(prog->aux); vfree(prog); return NULL; @@ -124,7 +124,7 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) for_each_possible_cpu(cpu) { struct bpf_prog_stats *pstats; - pstats = per_cpu_ptr(prog->aux->stats, cpu); + pstats = per_cpu_ptr(prog->stats, cpu); u64_stats_init(&pstats->syncp); } return prog; @@ -249,10 +249,10 @@ void __bpf_prog_free(struct bpf_prog *fp) if (fp->aux) { mutex_destroy(&fp->aux->used_maps_mutex); mutex_destroy(&fp->aux->dst_mutex); - free_percpu(fp->aux->stats); kfree(fp->aux->poke_tab); kfree(fp->aux); } + free_percpu(fp->stats); vfree(fp); } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index e5999d86c76e..f7df56a704de 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1739,7 +1739,7 @@ static void bpf_prog_get_stats(const struct bpf_prog *prog, unsigned int start; u64 tnsecs, tcnt; - st = per_cpu_ptr(prog->aux->stats, cpu); + st = per_cpu_ptr(prog->stats, cpu); do { start = u64_stats_fetch_begin_irq(&st->syncp); tnsecs = st->nsecs; diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 35c5887d82ff..5be3beeedd74 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -412,7 +412,7 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) * Hence check that 'start' is not zero. */ start) { - stats = this_cpu_ptr(prog->aux->stats); + stats = this_cpu_ptr(prog->stats); u64_stats_update_begin(&stats->syncp); stats->cnt++; stats->nsecs += sched_clock() - start; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 400d79e99fc8..424c1ba0f52f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11253,7 +11253,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) /* BPF_PROG_RUN doesn't call subprogs directly, * hence main prog stats include the runtime of subprogs. * subprogs don't have IDs and not reachable via prog_get_next_id - * func[i]->aux->stats will never be accessed and stays NULL + * func[i]->stats will never be accessed and stays NULL */ func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); if (!func[i]) -- cgit v1.2.3 From f2dd3b39467411c53703125a111f45b3672c1771 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 9 Feb 2021 19:36:28 -0800 Subject: bpf: Compute program stats for sleepable programs Since sleepable programs don't migrate from the cpu the excution stats can be computed for them as well. Reuse the same infrastructure for both sleepable and non-sleepable programs. run_cnt -> the number of times the program was executed. run_time_ns -> the program execution time in nanoseconds including the off-cpu time when the program was sleeping. Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: Andrii Nakryiko Acked-by: KP Singh Link: https://lore.kernel.org/bpf/20210210033634.62081-4-alexei.starovoitov@gmail.com --- arch/x86/net/bpf_jit_comp.c | 31 ++++++++++++------------------- include/linux/bpf.h | 4 ++-- kernel/bpf/trampoline.c | 42 ++++++++++++++++++++++++++++-------------- 3 files changed, 42 insertions(+), 35 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index a3dc3bd154ac..d11b9bcebbea 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1742,15 +1742,12 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, u8 *prog = *pprog; int cnt = 0; - if (p->aux->sleepable) { - if (emit_call(&prog, __bpf_prog_enter_sleepable, prog)) + if (emit_call(&prog, + p->aux->sleepable ? __bpf_prog_enter_sleepable : + __bpf_prog_enter, prog)) return -EINVAL; - } else { - if (emit_call(&prog, __bpf_prog_enter, prog)) - return -EINVAL; - /* remember prog start time returned by __bpf_prog_enter */ - emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0); - } + /* remember prog start time returned by __bpf_prog_enter */ + emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0); /* arg1: lea rdi, [rbp - stack_size] */ EMIT4(0x48, 0x8D, 0x7D, -stack_size); @@ -1770,18 +1767,14 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, if (mod_ret) emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); - if (p->aux->sleepable) { - if (emit_call(&prog, __bpf_prog_exit_sleepable, prog)) + /* arg1: mov rdi, progs[i] */ + emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); + /* arg2: mov rsi, rbx <- start time in nsec */ + emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6); + if (emit_call(&prog, + p->aux->sleepable ? __bpf_prog_exit_sleepable : + __bpf_prog_exit, prog)) return -EINVAL; - } else { - /* arg1: mov rdi, progs[i] */ - emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, - (u32) (long) p); - /* arg2: mov rsi, rbx <- start time in nsec */ - emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6); - if (emit_call(&prog, __bpf_prog_exit, prog)) - return -EINVAL; - } *pprog = prog; return 0; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 5a388955e6b6..e1840ceaf55f 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -563,8 +563,8 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, /* these two functions are called from generated trampoline */ u64 notrace __bpf_prog_enter(void); void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start); -void notrace __bpf_prog_enter_sleepable(void); -void notrace __bpf_prog_exit_sleepable(void); +u64 notrace __bpf_prog_enter_sleepable(void); +void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start); struct bpf_ksym { unsigned long start; diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 89fc849ba271..48eb021e1421 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -381,56 +381,70 @@ out: mutex_unlock(&trampoline_mutex); } +#define NO_START_TIME 0 +static u64 notrace bpf_prog_start_time(void) +{ + u64 start = NO_START_TIME; + + if (static_branch_unlikely(&bpf_stats_enabled_key)) + start = sched_clock(); + return start; +} + /* The logic is similar to BPF_PROG_RUN, but with an explicit * rcu_read_lock() and migrate_disable() which are required * for the trampoline. The macro is split into - * call _bpf_prog_enter + * call __bpf_prog_enter * call prog->bpf_func * call __bpf_prog_exit */ u64 notrace __bpf_prog_enter(void) __acquires(RCU) { - u64 start = 0; - rcu_read_lock(); migrate_disable(); - if (static_branch_unlikely(&bpf_stats_enabled_key)) - start = sched_clock(); - return start; + return bpf_prog_start_time(); } -void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) - __releases(RCU) +static void notrace update_prog_stats(struct bpf_prog *prog, + u64 start) { struct bpf_prog_stats *stats; if (static_branch_unlikely(&bpf_stats_enabled_key) && - /* static_key could be enabled in __bpf_prog_enter - * and disabled in __bpf_prog_exit. + /* static_key could be enabled in __bpf_prog_enter* + * and disabled in __bpf_prog_exit*. * And vice versa. - * Hence check that 'start' is not zero. + * Hence check that 'start' is valid. */ - start) { + start > NO_START_TIME) { stats = this_cpu_ptr(prog->stats); u64_stats_update_begin(&stats->syncp); stats->cnt++; stats->nsecs += sched_clock() - start; u64_stats_update_end(&stats->syncp); } +} + +void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) + __releases(RCU) +{ + update_prog_stats(prog, start); migrate_enable(); rcu_read_unlock(); } -void notrace __bpf_prog_enter_sleepable(void) +u64 notrace __bpf_prog_enter_sleepable(void) { rcu_read_lock_trace(); migrate_disable(); might_fault(); + return bpf_prog_start_time(); } -void notrace __bpf_prog_exit_sleepable(void) +void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start) { + update_prog_stats(prog, start); migrate_enable(); rcu_read_unlock_trace(); } -- cgit v1.2.3 From ca06f55b90020cd97f4cc6d52db95436162e7dcf Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 9 Feb 2021 19:36:29 -0800 Subject: bpf: Add per-program recursion prevention mechanism Since both sleepable and non-sleepable programs execute under migrate_disable add recursion prevention mechanism to both types of programs when they're executed via bpf trampoline. Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20210210033634.62081-5-alexei.starovoitov@gmail.com --- arch/x86/net/bpf_jit_comp.c | 15 ++++++++++++++ include/linux/bpf.h | 6 +++--- include/linux/filter.h | 1 + kernel/bpf/core.c | 8 ++++++++ kernel/bpf/trampoline.c | 23 ++++++++++++++++++---- .../selftests/bpf/prog_tests/fexit_stress.c | 4 ++-- .../selftests/bpf/prog_tests/trampoline_count.c | 4 ++-- 7 files changed, 50 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d11b9bcebbea..79e7a0ec1da5 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1740,8 +1740,11 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, struct bpf_prog *p, int stack_size, bool mod_ret) { u8 *prog = *pprog; + u8 *jmp_insn; int cnt = 0; + /* arg1: mov rdi, progs[i] */ + emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); if (emit_call(&prog, p->aux->sleepable ? __bpf_prog_enter_sleepable : __bpf_prog_enter, prog)) @@ -1749,6 +1752,14 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, /* remember prog start time returned by __bpf_prog_enter */ emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0); + /* if (__bpf_prog_enter*(prog) == 0) + * goto skip_exec_of_prog; + */ + EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */ + /* emit 2 nops that will be replaced with JE insn */ + jmp_insn = prog; + emit_nops(&prog, 2); + /* arg1: lea rdi, [rbp - stack_size] */ EMIT4(0x48, 0x8D, 0x7D, -stack_size); /* arg2: progs[i]->insnsi for interpreter */ @@ -1767,6 +1778,10 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, if (mod_ret) emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); + /* replace 2 nops with JE insn, since jmp target is known */ + jmp_insn[0] = X86_JE; + jmp_insn[1] = prog - jmp_insn - 2; + /* arg1: mov rdi, progs[i] */ emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); /* arg2: mov rsi, rbx <- start time in nsec */ diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e1840ceaf55f..1c8ea682c007 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -529,7 +529,7 @@ struct btf_func_model { /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2 */ -#define BPF_MAX_TRAMP_PROGS 40 +#define BPF_MAX_TRAMP_PROGS 38 struct bpf_tramp_progs { struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS]; @@ -561,9 +561,9 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end, struct bpf_tramp_progs *tprogs, void *orig_call); /* these two functions are called from generated trampoline */ -u64 notrace __bpf_prog_enter(void); +u64 notrace __bpf_prog_enter(struct bpf_prog *prog); void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start); -u64 notrace __bpf_prog_enter_sleepable(void); +u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog); void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start); struct bpf_ksym { diff --git a/include/linux/filter.h b/include/linux/filter.h index cecb03c9d251..6a06f3c69f4e 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -565,6 +565,7 @@ struct bpf_prog { u32 jited_len; /* Size of jited insns in bytes */ u8 tag[BPF_TAG_SIZE]; struct bpf_prog_stats __percpu *stats; + int __percpu *active; unsigned int (*bpf_func)(const void *ctx, const struct bpf_insn *insn); struct bpf_prog_aux *aux; /* Auxiliary fields */ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 2cf71fd39c22..334070c4b8a1 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -91,6 +91,12 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag vfree(fp); return NULL; } + fp->active = alloc_percpu_gfp(int, GFP_KERNEL_ACCOUNT | gfp_extra_flags); + if (!fp->active) { + vfree(fp); + kfree(aux); + return NULL; + } fp->pages = size / PAGE_SIZE; fp->aux = aux; @@ -116,6 +122,7 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); if (!prog->stats) { + free_percpu(prog->active); kfree(prog->aux); vfree(prog); return NULL; @@ -253,6 +260,7 @@ void __bpf_prog_free(struct bpf_prog *fp) kfree(fp->aux); } free_percpu(fp->stats); + free_percpu(fp->active); vfree(fp); } diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 48eb021e1421..89ef6320d19b 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -381,13 +381,16 @@ out: mutex_unlock(&trampoline_mutex); } -#define NO_START_TIME 0 +#define NO_START_TIME 1 static u64 notrace bpf_prog_start_time(void) { u64 start = NO_START_TIME; - if (static_branch_unlikely(&bpf_stats_enabled_key)) + if (static_branch_unlikely(&bpf_stats_enabled_key)) { start = sched_clock(); + if (unlikely(!start)) + start = NO_START_TIME; + } return start; } @@ -397,12 +400,20 @@ static u64 notrace bpf_prog_start_time(void) * call __bpf_prog_enter * call prog->bpf_func * call __bpf_prog_exit + * + * __bpf_prog_enter returns: + * 0 - skip execution of the bpf prog + * 1 - execute bpf prog + * [2..MAX_U64] - excute bpf prog and record execution time. + * This is start time. */ -u64 notrace __bpf_prog_enter(void) +u64 notrace __bpf_prog_enter(struct bpf_prog *prog) __acquires(RCU) { rcu_read_lock(); migrate_disable(); + if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) + return 0; return bpf_prog_start_time(); } @@ -430,21 +441,25 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) __releases(RCU) { update_prog_stats(prog, start); + __this_cpu_dec(*(prog->active)); migrate_enable(); rcu_read_unlock(); } -u64 notrace __bpf_prog_enter_sleepable(void) +u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog) { rcu_read_lock_trace(); migrate_disable(); might_fault(); + if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) + return 0; return bpf_prog_start_time(); } void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start) { update_prog_stats(prog, start); + __this_cpu_dec(*(prog->active)); migrate_enable(); rcu_read_unlock_trace(); } diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_stress.c b/tools/testing/selftests/bpf/prog_tests/fexit_stress.c index 3b9dbf7433f0..7c9b62e971f1 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_stress.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_stress.c @@ -2,8 +2,8 @@ /* Copyright (c) 2019 Facebook */ #include -/* x86-64 fits 55 JITed and 43 interpreted progs into half page */ -#define CNT 40 +/* that's kernel internal BPF_MAX_TRAMP_PROGS define */ +#define CNT 38 void test_fexit_stress(void) { diff --git a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c index 781c8d11604b..f3022d934e2d 100644 --- a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c +++ b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c @@ -4,7 +4,7 @@ #include #include -#define MAX_TRAMP_PROGS 40 +#define MAX_TRAMP_PROGS 38 struct inst { struct bpf_object *obj; @@ -52,7 +52,7 @@ void test_trampoline_count(void) struct bpf_link *link; char comm[16] = {}; - /* attach 'allowed' 40 trampoline programs */ + /* attach 'allowed' trampoline programs */ for (i = 0; i < MAX_TRAMP_PROGS; i++) { obj = bpf_object__open_file(object, NULL); if (CHECK(IS_ERR(obj), "obj_open_file", "err %ld\n", PTR_ERR(obj))) { -- cgit v1.2.3 From 9ed9e9ba2337205311398a312796c213737bac35 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 9 Feb 2021 19:36:31 -0800 Subject: bpf: Count the number of times recursion was prevented Add per-program counter for number of times recursion prevention mechanism was triggered and expose it via show_fdinfo and bpf_prog_info. Teach bpftool to print it. Signed-off-by: Alexei Starovoitov Signed-off-by: Daniel Borkmann Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20210210033634.62081-7-alexei.starovoitov@gmail.com --- include/linux/filter.h | 1 + include/uapi/linux/bpf.h | 1 + kernel/bpf/syscall.c | 14 ++++++++++---- kernel/bpf/trampoline.c | 18 ++++++++++++++++-- tools/bpf/bpftool/prog.c | 4 ++++ tools/include/uapi/linux/bpf.h | 1 + 6 files changed, 33 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/filter.h b/include/linux/filter.h index 6a06f3c69f4e..3b00fc906ccd 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -543,6 +543,7 @@ struct bpf_binary_header { struct bpf_prog_stats { u64 cnt; u64 nsecs; + u64 misses; struct u64_stats_sync syncp; } __aligned(2 * sizeof(u64)); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index c001766adcbc..c547ad1ffe43 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4501,6 +4501,7 @@ struct bpf_prog_info { __aligned_u64 prog_tags; __u64 run_time_ns; __u64 run_cnt; + __u64 recursion_misses; } __attribute__((aligned(8))); struct bpf_map_info { diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index f7df56a704de..c859bc46d06c 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1731,25 +1731,28 @@ static int bpf_prog_release(struct inode *inode, struct file *filp) static void bpf_prog_get_stats(const struct bpf_prog *prog, struct bpf_prog_stats *stats) { - u64 nsecs = 0, cnt = 0; + u64 nsecs = 0, cnt = 0, misses = 0; int cpu; for_each_possible_cpu(cpu) { const struct bpf_prog_stats *st; unsigned int start; - u64 tnsecs, tcnt; + u64 tnsecs, tcnt, tmisses; st = per_cpu_ptr(prog->stats, cpu); do { start = u64_stats_fetch_begin_irq(&st->syncp); tnsecs = st->nsecs; tcnt = st->cnt; + tmisses = st->misses; } while (u64_stats_fetch_retry_irq(&st->syncp, start)); nsecs += tnsecs; cnt += tcnt; + misses += tmisses; } stats->nsecs = nsecs; stats->cnt = cnt; + stats->misses = misses; } #ifdef CONFIG_PROC_FS @@ -1768,14 +1771,16 @@ static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) "memlock:\t%llu\n" "prog_id:\t%u\n" "run_time_ns:\t%llu\n" - "run_cnt:\t%llu\n", + "run_cnt:\t%llu\n" + "recursion_misses:\t%llu\n", prog->type, prog->jited, prog_tag, prog->pages * 1ULL << PAGE_SHIFT, prog->aux->id, stats.nsecs, - stats.cnt); + stats.cnt, + stats.misses); } #endif @@ -3438,6 +3443,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, bpf_prog_get_stats(prog, &stats); info.run_time_ns = stats.nsecs; info.run_cnt = stats.cnt; + info.recursion_misses = stats.misses; if (!bpf_capable()) { info.jited_prog_len = 0; diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 89ef6320d19b..7bc3b3209224 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -394,6 +394,16 @@ static u64 notrace bpf_prog_start_time(void) return start; } +static void notrace inc_misses_counter(struct bpf_prog *prog) +{ + struct bpf_prog_stats *stats; + + stats = this_cpu_ptr(prog->stats); + u64_stats_update_begin(&stats->syncp); + stats->misses++; + u64_stats_update_end(&stats->syncp); +} + /* The logic is similar to BPF_PROG_RUN, but with an explicit * rcu_read_lock() and migrate_disable() which are required * for the trampoline. The macro is split into @@ -412,8 +422,10 @@ u64 notrace __bpf_prog_enter(struct bpf_prog *prog) { rcu_read_lock(); migrate_disable(); - if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) + if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) { + inc_misses_counter(prog); return 0; + } return bpf_prog_start_time(); } @@ -451,8 +463,10 @@ u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog) rcu_read_lock_trace(); migrate_disable(); might_fault(); - if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) + if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) { + inc_misses_counter(prog); return 0; + } return bpf_prog_start_time(); } diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 1fe3ba255bad..f2b915b20546 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -368,6 +368,8 @@ static void print_prog_header_json(struct bpf_prog_info *info) jsonw_uint_field(json_wtr, "run_time_ns", info->run_time_ns); jsonw_uint_field(json_wtr, "run_cnt", info->run_cnt); } + if (info->recursion_misses) + jsonw_uint_field(json_wtr, "recursion_misses", info->recursion_misses); } static void print_prog_json(struct bpf_prog_info *info, int fd) @@ -446,6 +448,8 @@ static void print_prog_header_plain(struct bpf_prog_info *info) if (info->run_time_ns) printf(" run_time_ns %lld run_cnt %lld", info->run_time_ns, info->run_cnt); + if (info->recursion_misses) + printf(" recursion_misses %lld", info->recursion_misses); printf("\n"); } diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index c001766adcbc..c547ad1ffe43 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -4501,6 +4501,7 @@ struct bpf_prog_info { __aligned_u64 prog_tags; __u64 run_time_ns; __u64 run_cnt; + __u64 recursion_misses; } __attribute__((aligned(8))); struct bpf_map_info { -- cgit v1.2.3 From 4217a64e18a1647a0dbc68cb3169a5a06f054ec8 Mon Sep 17 00:00:00 2001 From: Michael Walle Date: Tue, 9 Feb 2021 17:38:52 +0100 Subject: net: phy: introduce phydev->port At the moment, PORT_MII is reported in the ethtool ops. This is odd because it is an interface between the MAC and the PHY and no external port. Some network card drivers will overwrite the port to twisted pair or fiber, though. Even worse, the MDI/MDIX setting is only used by ethtool if the port is twisted pair. Set the port to PORT_TP by default because most PHY drivers are copper ones. If there is fibre support and it is enabled, the PHY driver will set it to PORT_FIBRE. This will change reporting PORT_MII to either PORT_TP or PORT_FIBRE; except for the genphy fallback driver. Suggested-by: Andrew Lunn Signed-off-by: Michael Walle Reviewed-by: Florian Fainelli Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- drivers/net/phy/broadcom.c | 2 ++ drivers/net/phy/dp83822.c | 3 +++ drivers/net/phy/dp83869.c | 4 ++++ drivers/net/phy/lxt.c | 1 + drivers/net/phy/marvell.c | 1 + drivers/net/phy/marvell10g.c | 2 ++ drivers/net/phy/micrel.c | 14 +++++++++++--- drivers/net/phy/phy.c | 2 +- drivers/net/phy/phy_device.c | 9 +++++++++ include/linux/phy.h | 2 ++ 10 files changed, 36 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c index 3142ba768313..0472b3470c59 100644 --- a/drivers/net/phy/broadcom.c +++ b/drivers/net/phy/broadcom.c @@ -410,6 +410,8 @@ static int bcm54616s_probe(struct phy_device *phydev) */ if (!(val & BCM54616S_100FX_MODE)) phydev->dev_flags |= PHY_BCM_FLAGS_MODE_1000BX; + + phydev->port = PORT_FIBRE; } return 0; diff --git a/drivers/net/phy/dp83822.c b/drivers/net/phy/dp83822.c index fff371ca1086..be1224b4447b 100644 --- a/drivers/net/phy/dp83822.c +++ b/drivers/net/phy/dp83822.c @@ -554,6 +554,9 @@ static int dp83822_probe(struct phy_device *phydev) dp83822_of_init(phydev); + if (dp83822->fx_enabled) + phydev->port = PORT_FIBRE; + return 0; } diff --git a/drivers/net/phy/dp83869.c b/drivers/net/phy/dp83869.c index b30bc142d82e..755220c6451f 100644 --- a/drivers/net/phy/dp83869.c +++ b/drivers/net/phy/dp83869.c @@ -855,6 +855,10 @@ static int dp83869_probe(struct phy_device *phydev) if (ret) return ret; + if (dp83869->mode == DP83869_RGMII_100_BASE || + dp83869->mode == DP83869_RGMII_1000_BASE) + phydev->port = PORT_FIBRE; + return dp83869_config_init(phydev); } diff --git a/drivers/net/phy/lxt.c b/drivers/net/phy/lxt.c index 0ee23d29c0d4..bde3356a2f86 100644 --- a/drivers/net/phy/lxt.c +++ b/drivers/net/phy/lxt.c @@ -292,6 +292,7 @@ static int lxt973_probe(struct phy_device *phydev) phy_write(phydev, MII_BMCR, val); /* Remember that the port is in fiber mode. */ phydev->priv = lxt973_probe; + phydev->port = PORT_FIBRE; } else { phydev->priv = NULL; } diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c index b523aa37ebf0..3238d0fbf437 100644 --- a/drivers/net/phy/marvell.c +++ b/drivers/net/phy/marvell.c @@ -1552,6 +1552,7 @@ static int marvell_read_status_page(struct phy_device *phydev, int page) phydev->asym_pause = 0; phydev->speed = SPEED_UNKNOWN; phydev->duplex = DUPLEX_UNKNOWN; + phydev->port = fiber ? PORT_FIBRE : PORT_TP; if (phydev->autoneg == AUTONEG_ENABLE) err = marvell_read_status_page_an(phydev, fiber, status); diff --git a/drivers/net/phy/marvell10g.c b/drivers/net/phy/marvell10g.c index 1901ba277413..b1bb9b8e1e4e 100644 --- a/drivers/net/phy/marvell10g.c +++ b/drivers/net/phy/marvell10g.c @@ -631,6 +631,7 @@ static int mv3310_read_status_10gbaser(struct phy_device *phydev) phydev->link = 1; phydev->speed = SPEED_10000; phydev->duplex = DUPLEX_FULL; + phydev->port = PORT_FIBRE; return 0; } @@ -690,6 +691,7 @@ static int mv3310_read_status_copper(struct phy_device *phydev) phydev->duplex = cssr1 & MV_PCS_CSSR1_DUPLEX_FULL ? DUPLEX_FULL : DUPLEX_HALF; + phydev->port = PORT_TP; phydev->mdix = cssr1 & MV_PCS_CSSR1_MDIX ? ETH_TP_MDI_X : ETH_TP_MDI; diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index 494abf608b8f..7ec6f70d6a82 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -341,14 +341,19 @@ static int kszphy_config_init(struct phy_device *phydev) return kszphy_config_reset(phydev); } +static int ksz8041_fiber_mode(struct phy_device *phydev) +{ + struct device_node *of_node = phydev->mdio.dev.of_node; + + return of_property_read_bool(of_node, "micrel,fiber-mode"); +} + static int ksz8041_config_init(struct phy_device *phydev) { __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - struct device_node *of_node = phydev->mdio.dev.of_node; - /* Limit supported and advertised modes in fiber mode */ - if (of_property_read_bool(of_node, "micrel,fiber-mode")) { + if (ksz8041_fiber_mode(phydev)) { phydev->dev_flags |= MICREL_PHY_FXEN; linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, mask); linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Half_BIT, mask); @@ -1176,6 +1181,9 @@ static int kszphy_probe(struct phy_device *phydev) } } + if (ksz8041_fiber_mode(phydev)) + phydev->port = PORT_FIBRE; + /* Support legacy board-file configuration */ if (phydev->dev_flags & MICREL_PHY_50MHZ_CLK) { priv->rmii_ref_clk_sel = true; diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index 9cb7e4dbf8f4..fdb914b5b857 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -308,7 +308,7 @@ void phy_ethtool_ksettings_get(struct phy_device *phydev, if (phydev->interface == PHY_INTERFACE_MODE_MOCA) cmd->base.port = PORT_BNC; else - cmd->base.port = PORT_MII; + cmd->base.port = phydev->port; cmd->base.transceiver = phy_is_internal(phydev) ? XCVR_INTERNAL : XCVR_EXTERNAL; cmd->base.phy_address = phydev->mdio.addr; diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 8447e56ba572..30a20a29ae05 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -606,6 +606,7 @@ struct phy_device *phy_device_create(struct mii_bus *bus, int addr, u32 phy_id, dev->pause = 0; dev->asym_pause = 0; dev->link = 0; + dev->port = PORT_TP; dev->interface = PHY_INTERFACE_MODE_GMII; dev->autoneg = AUTONEG_ENABLE; @@ -1403,6 +1404,14 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev, phydev->state = PHY_READY; + /* Port is set to PORT_TP by default and the actual PHY driver will set + * it to different value depending on the PHY configuration. If we have + * the generic PHY driver we can't figure it out, thus set the old + * legacy PORT_MII value. + */ + if (using_genphy) + phydev->port = PORT_MII; + /* Initial carrier state is off as the phy is about to be * (re)initialized. */ diff --git a/include/linux/phy.h b/include/linux/phy.h index bc323fbdd21e..c130788306c8 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -503,6 +503,7 @@ struct macsec_ops; * * @speed: Current link speed * @duplex: Current duplex + * @port: Current port * @pause: Current pause * @asym_pause: Current asymmetric pause * @supported: Combined MAC/PHY supported linkmodes @@ -581,6 +582,7 @@ struct phy_device { */ int speed; int duplex; + int port; int pause; int asym_pause; u8 master_slave_get; -- cgit v1.2.3 From dcf0cd1cc58b8e88793ad6531db9b3a47324ca09 Mon Sep 17 00:00:00 2001 From: George McCollister Date: Tue, 9 Feb 2021 19:02:11 -0600 Subject: net: hsr: add offloading support Add support for offloading of HSR/PRP (IEC 62439-3) tag insertion tag removal, duplicate generation and forwarding. For HSR, insertion involves the switch adding a 6 byte HSR header after the 14 byte Ethernet header. For PRP it adds a 6 byte trailer. Tag removal involves automatically stripping the HSR/PRP header/trailer in the switch. This is possible when the switch also performs auto deduplication using the HSR/PRP header/trailer (making it no longer required). Forwarding involves automatically forwarding between redundant ports in an HSR. This is crucial because delay is accumulated as a frame passes through each node in the ring. Duplication involves the switch automatically sending a single frame from the CPU port to both redundant ports. This is required because the inserted HSR/PRP header/trailer must contain the same sequence number on the frames sent out both redundant ports. Export is_hsr_master so DSA can tell them apart from other devices in dsa_slave_changeupper. Signed-off-by: George McCollister Reviewed-by: Vladimir Oltean Signed-off-by: David S. Miller --- Documentation/networking/netdev-features.rst | 21 +++++++++++++++++++++ include/linux/if_hsr.h | 27 +++++++++++++++++++++++++++ include/linux/netdev_features.h | 9 +++++++++ net/ethtool/common.c | 4 ++++ net/hsr/hsr_device.c | 14 +++----------- net/hsr/hsr_device.h | 1 - net/hsr/hsr_forward.c | 27 ++++++++++++++++++++++++--- net/hsr/hsr_forward.h | 1 + net/hsr/hsr_framereg.c | 2 ++ net/hsr/hsr_main.c | 11 +++++++++++ net/hsr/hsr_main.h | 8 +------- net/hsr/hsr_slave.c | 10 ++++++---- 12 files changed, 109 insertions(+), 26 deletions(-) create mode 100644 include/linux/if_hsr.h (limited to 'include/linux') diff --git a/Documentation/networking/netdev-features.rst b/Documentation/networking/netdev-features.rst index a2d7d7160e39..d7b15bb64deb 100644 --- a/Documentation/networking/netdev-features.rst +++ b/Documentation/networking/netdev-features.rst @@ -182,3 +182,24 @@ stricter than Hardware LRO. A packet stream merged by Hardware GRO must be re-segmentable by GSO or TSO back to the exact original packet stream. Hardware GRO is dependent on RXCSUM since every packet successfully merged by hardware must also have the checksum verified by hardware. + +* hsr-tag-ins-offload + +This should be set for devices which insert an HSR (High-availability Seamless +Redundancy) or PRP (Parallel Redundancy Protocol) tag automatically. + +* hsr-tag-rm-offload + +This should be set for devices which remove HSR (High-availability Seamless +Redundancy) or PRP (Parallel Redundancy Protocol) tags automatically. + +* hsr-fwd-offload + +This should be set for devices which forward HSR (High-availability Seamless +Redundancy) frames from one port to another in hardware. + +* hsr-dup-offload + +This should be set for devices which duplicate outgoing HSR (High-availability +Seamless Redundancy) or PRP (Parallel Redundancy Protocol) tags automatically +frames in hardware. diff --git a/include/linux/if_hsr.h b/include/linux/if_hsr.h new file mode 100644 index 000000000000..38bbc537d4e4 --- /dev/null +++ b/include/linux/if_hsr.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IF_HSR_H_ +#define _LINUX_IF_HSR_H_ + +/* used to differentiate various protocols */ +enum hsr_version { + HSR_V0 = 0, + HSR_V1, + PRP_V1, +}; + +#if IS_ENABLED(CONFIG_HSR) +extern bool is_hsr_master(struct net_device *dev); +extern int hsr_get_version(struct net_device *dev, enum hsr_version *ver); +#else +static inline bool is_hsr_master(struct net_device *dev) +{ + return false; +} +static inline int hsr_get_version(struct net_device *dev, + enum hsr_version *ver) +{ + return -EINVAL; +} +#endif /* CONFIG_HSR */ + +#endif /*_LINUX_IF_HSR_H_*/ diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h index c06d6aaba9df..3de38d6a0aea 100644 --- a/include/linux/netdev_features.h +++ b/include/linux/netdev_features.h @@ -86,6 +86,11 @@ enum { NETIF_F_HW_MACSEC_BIT, /* Offload MACsec operations */ NETIF_F_GRO_UDP_FWD_BIT, /* Allow UDP GRO for forwarding */ + NETIF_F_HW_HSR_TAG_INS_BIT, /* Offload HSR tag insertion */ + NETIF_F_HW_HSR_TAG_RM_BIT, /* Offload HSR tag removal */ + NETIF_F_HW_HSR_FWD_BIT, /* Offload HSR forwarding */ + NETIF_F_HW_HSR_DUP_BIT, /* Offload HSR duplication */ + /* * Add your fresh new feature above and remember to update * netdev_features_strings[] in net/core/ethtool.c and maybe @@ -159,6 +164,10 @@ enum { #define NETIF_F_GSO_FRAGLIST __NETIF_F(GSO_FRAGLIST) #define NETIF_F_HW_MACSEC __NETIF_F(HW_MACSEC) #define NETIF_F_GRO_UDP_FWD __NETIF_F(GRO_UDP_FWD) +#define NETIF_F_HW_HSR_TAG_INS __NETIF_F(HW_HSR_TAG_INS) +#define NETIF_F_HW_HSR_TAG_RM __NETIF_F(HW_HSR_TAG_RM) +#define NETIF_F_HW_HSR_FWD __NETIF_F(HW_HSR_FWD) +#define NETIF_F_HW_HSR_DUP __NETIF_F(HW_HSR_DUP) /* Finds the next feature with the highest number of the range of start till 0. */ diff --git a/net/ethtool/common.c b/net/ethtool/common.c index 835b9bba3e7e..c6a383dfd6c2 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -69,6 +69,10 @@ const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN] = { [NETIF_F_GRO_FRAGLIST_BIT] = "rx-gro-list", [NETIF_F_HW_MACSEC_BIT] = "macsec-hw-offload", [NETIF_F_GRO_UDP_FWD_BIT] = "rx-udp-gro-forwarding", + [NETIF_F_HW_HSR_TAG_INS_BIT] = "hsr-tag-ins-offload", + [NETIF_F_HW_HSR_TAG_RM_BIT] = "hsr-tag-rm-offload", + [NETIF_F_HW_HSR_FWD_BIT] = "hsr-fwd-offload", + [NETIF_F_HW_HSR_DUP_BIT] = "hsr-dup-offload", }; const char diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c index ec6a68b403d5..7444ec6e298e 100644 --- a/net/hsr/hsr_device.c +++ b/net/hsr/hsr_device.c @@ -417,6 +417,7 @@ static struct hsr_proto_ops hsr_ops = { .send_sv_frame = send_hsr_supervision_frame, .create_tagged_frame = hsr_create_tagged_frame, .get_untagged_frame = hsr_get_untagged_frame, + .drop_frame = hsr_drop_frame, .fill_frame_info = hsr_fill_frame_info, .invalid_dan_ingress_frame = hsr_invalid_dan_ingress_frame, }; @@ -464,10 +465,11 @@ void hsr_dev_setup(struct net_device *dev) /* Return true if dev is a HSR master; return false otherwise. */ -inline bool is_hsr_master(struct net_device *dev) +bool is_hsr_master(struct net_device *dev) { return (dev->netdev_ops->ndo_start_xmit == hsr_dev_xmit); } +EXPORT_SYMBOL(is_hsr_master); /* Default multicast address for HSR Supervision frames */ static const unsigned char def_multicast_addr[ETH_ALEN] __aligned(2) = { @@ -520,16 +522,6 @@ int hsr_dev_finalize(struct net_device *hsr_dev, struct net_device *slave[2], hsr->prot_version = protocol_version; - /* FIXME: should I modify the value of these? - * - * - hsr_dev->flags - i.e. - * IFF_MASTER/SLAVE? - * - hsr_dev->priv_flags - i.e. - * IFF_EBRIDGE? - * IFF_TX_SKB_SHARING? - * IFF_HSR_MASTER/SLAVE? - */ - /* Make sure the 1st call to netif_carrier_on() gets through */ netif_carrier_off(hsr_dev); diff --git a/net/hsr/hsr_device.h b/net/hsr/hsr_device.h index 868373822ee4..9060c92168f9 100644 --- a/net/hsr/hsr_device.h +++ b/net/hsr/hsr_device.h @@ -19,6 +19,5 @@ int hsr_dev_finalize(struct net_device *hsr_dev, struct net_device *slave[2], unsigned char multicast_spec, u8 protocol_version, struct netlink_ext_ack *extack); void hsr_check_carrier_and_operstate(struct hsr_priv *hsr); -bool is_hsr_master(struct net_device *dev); int hsr_get_max_mtu(struct hsr_priv *hsr); #endif /* __HSR_DEVICE_H */ diff --git a/net/hsr/hsr_forward.c b/net/hsr/hsr_forward.c index d32cd87d5c5b..ed82a470b6e1 100644 --- a/net/hsr/hsr_forward.c +++ b/net/hsr/hsr_forward.c @@ -249,6 +249,8 @@ struct sk_buff *hsr_create_tagged_frame(struct hsr_frame_info *frame, /* set the lane id properly */ hsr_set_path_id(hsr_ethhdr, port); return skb_clone(frame->skb_hsr, GFP_ATOMIC); + } else if (port->dev->features & NETIF_F_HW_HSR_TAG_INS) { + return skb_clone(frame->skb_std, GFP_ATOMIC); } /* Create the new skb with enough headroom to fit the HSR tag */ @@ -291,6 +293,8 @@ struct sk_buff *prp_create_tagged_frame(struct hsr_frame_info *frame, return NULL; } return skb_clone(frame->skb_prp, GFP_ATOMIC); + } else if (port->dev->features & NETIF_F_HW_HSR_TAG_INS) { + return skb_clone(frame->skb_std, GFP_ATOMIC); } skb = skb_copy_expand(frame->skb_std, 0, @@ -343,6 +347,14 @@ bool prp_drop_frame(struct hsr_frame_info *frame, struct hsr_port *port) port->type == HSR_PT_SLAVE_A)); } +bool hsr_drop_frame(struct hsr_frame_info *frame, struct hsr_port *port) +{ + if (port->dev->features & NETIF_F_HW_HSR_FWD) + return prp_drop_frame(frame, port); + + return false; +} + /* Forward the frame through all devices except: * - Back through the receiving device * - If it's a HSR frame: through a device where it has passed before @@ -359,6 +371,7 @@ static void hsr_forward_do(struct hsr_frame_info *frame) { struct hsr_port *port; struct sk_buff *skb; + bool sent = false; hsr_for_each_port(frame->port_rcv->hsr, port) { struct hsr_priv *hsr = port->hsr; @@ -374,6 +387,12 @@ static void hsr_forward_do(struct hsr_frame_info *frame) if (port->type != HSR_PT_MASTER && frame->is_local_exclusive) continue; + /* If hardware duplicate generation is enabled, only send out + * one port. + */ + if ((port->dev->features & NETIF_F_HW_HSR_DUP) && sent) + continue; + /* Don't send frame over port where it has been sent before. * Also fro SAN, this shouldn't be done. */ @@ -405,10 +424,12 @@ static void hsr_forward_do(struct hsr_frame_info *frame) } skb->dev = port->dev; - if (port->type == HSR_PT_MASTER) + if (port->type == HSR_PT_MASTER) { hsr_deliver_master(skb, port->dev, frame->node_src); - else - hsr_xmit(skb, port, frame); + } else { + if (!hsr_xmit(skb, port, frame)) + sent = true; + } } } diff --git a/net/hsr/hsr_forward.h b/net/hsr/hsr_forward.h index 618140d484ad..b6acaafa83fc 100644 --- a/net/hsr/hsr_forward.h +++ b/net/hsr/hsr_forward.h @@ -23,6 +23,7 @@ struct sk_buff *hsr_get_untagged_frame(struct hsr_frame_info *frame, struct sk_buff *prp_get_untagged_frame(struct hsr_frame_info *frame, struct hsr_port *port); bool prp_drop_frame(struct hsr_frame_info *frame, struct hsr_port *port); +bool hsr_drop_frame(struct hsr_frame_info *frame, struct hsr_port *port); void prp_fill_frame_info(__be16 proto, struct sk_buff *skb, struct hsr_frame_info *frame); void hsr_fill_frame_info(__be16 proto, struct sk_buff *skb, diff --git a/net/hsr/hsr_framereg.c b/net/hsr/hsr_framereg.c index 5c97de459905..f9a8cc82ae2e 100644 --- a/net/hsr/hsr_framereg.c +++ b/net/hsr/hsr_framereg.c @@ -277,6 +277,8 @@ void hsr_handle_sup_frame(struct hsr_frame_info *frame) skb = frame->skb_hsr; else if (frame->skb_prp) skb = frame->skb_prp; + else if (frame->skb_std) + skb = frame->skb_std; if (!skb) return; diff --git a/net/hsr/hsr_main.c b/net/hsr/hsr_main.c index 2fd1976e5b1c..f7e284f23b1f 100644 --- a/net/hsr/hsr_main.c +++ b/net/hsr/hsr_main.c @@ -131,6 +131,17 @@ struct hsr_port *hsr_port_get_hsr(struct hsr_priv *hsr, enum hsr_port_type pt) return NULL; } +int hsr_get_version(struct net_device *dev, enum hsr_version *ver) +{ + struct hsr_priv *hsr; + + hsr = netdev_priv(dev); + *ver = hsr->prot_version; + + return 0; +} +EXPORT_SYMBOL(hsr_get_version); + static struct notifier_block hsr_nb = { .notifier_call = hsr_netdev_notify, /* Slave event notifications */ }; diff --git a/net/hsr/hsr_main.h b/net/hsr/hsr_main.h index a9c30a608e35..a169808ee78a 100644 --- a/net/hsr/hsr_main.h +++ b/net/hsr/hsr_main.h @@ -13,6 +13,7 @@ #include #include #include +#include /* Time constants as specified in the HSR specification (IEC-62439-3 2010) * Table 8. @@ -171,13 +172,6 @@ struct hsr_port { enum hsr_port_type type; }; -/* used by driver internally to differentiate various protocols */ -enum hsr_version { - HSR_V0 = 0, - HSR_V1, - PRP_V1, -}; - struct hsr_frame_info; struct hsr_node; diff --git a/net/hsr/hsr_slave.c b/net/hsr/hsr_slave.c index 36d5fcf09c61..c5227d42faf5 100644 --- a/net/hsr/hsr_slave.c +++ b/net/hsr/hsr_slave.c @@ -48,12 +48,14 @@ static rx_handler_result_t hsr_handle_frame(struct sk_buff **pskb) goto finish_consume; } - /* For HSR, only tagged frames are expected, but for PRP - * there could be non tagged frames as well from Single - * attached nodes (SANs). + /* For HSR, only tagged frames are expected (unless the device offloads + * HSR tag removal), but for PRP there could be non tagged frames as + * well from Single attached nodes (SANs). */ protocol = eth_hdr(skb)->h_proto; - if (hsr->proto_ops->invalid_dan_ingress_frame && + + if (!(port->dev->features & NETIF_F_HW_HSR_TAG_RM) && + hsr->proto_ops->invalid_dan_ingress_frame && hsr->proto_ops->invalid_dan_ingress_frame(protocol)) goto finish_pass; -- cgit v1.2.3 From 4c236d5dc8b86222dc155cd68e7934624264150f Mon Sep 17 00:00:00 2001 From: Geetha sowjanya Date: Thu, 11 Feb 2021 21:28:27 +0530 Subject: octeontx2-pf: cn10k: Use LMTST lines for NPA/NIX operations This patch adds support to use new LMTST lines for NPA batch free and burst SQE flush. Adds new dev_hw_ops structure to hold platform specific functions and create new files cn10k.c and cn10k.h. Signed-off-by: Geetha sowjanya Signed-off-by: Sunil Goutham Signed-off-by: David S. Miller --- .../net/ethernet/marvell/octeontx2/nic/Makefile | 2 +- drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c | 181 +++++++++++++++++++++ drivers/net/ethernet/marvell/octeontx2/nic/cn10k.h | 17 ++ .../ethernet/marvell/octeontx2/nic/otx2_common.c | 84 ++++------ .../ethernet/marvell/octeontx2/nic/otx2_common.h | 68 +++++++- .../net/ethernet/marvell/octeontx2/nic/otx2_pf.c | 36 +--- .../net/ethernet/marvell/octeontx2/nic/otx2_reg.h | 1 + .../net/ethernet/marvell/octeontx2/nic/otx2_txrx.c | 39 ++--- .../net/ethernet/marvell/octeontx2/nic/otx2_txrx.h | 7 + .../net/ethernet/marvell/octeontx2/nic/otx2_vf.c | 28 +--- include/linux/soc/marvell/octeontx2/asm.h | 8 + 11 files changed, 336 insertions(+), 135 deletions(-) create mode 100644 drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c create mode 100644 drivers/net/ethernet/marvell/octeontx2/nic/cn10k.h (limited to 'include/linux') diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile index 29c82b94b2dc..745aa8a19499 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile +++ b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile @@ -7,7 +7,7 @@ obj-$(CONFIG_OCTEONTX2_PF) += rvu_nicpf.o obj-$(CONFIG_OCTEONTX2_VF) += rvu_nicvf.o rvu_nicpf-y := otx2_pf.o otx2_common.o otx2_txrx.o otx2_ethtool.o \ - otx2_ptp.o otx2_flows.o + otx2_ptp.o otx2_flows.o cn10k.o rvu_nicvf-y := otx2_vf.o ccflags-y += -I$(srctree)/drivers/net/ethernet/marvell/octeontx2/af diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c new file mode 100644 index 000000000000..d6ca809edaed --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell OcteonTx2 RVU Physcial Function ethernet driver + * + * Copyright (C) 2020 Marvell. + */ + +#include "cn10k.h" +#include "otx2_reg.h" +#include "otx2_struct.h" + +static struct dev_hw_ops otx2_hw_ops = { + .sq_aq_init = otx2_sq_aq_init, + .sqe_flush = otx2_sqe_flush, + .aura_freeptr = otx2_aura_freeptr, + .refill_pool_ptrs = otx2_refill_pool_ptrs, +}; + +static struct dev_hw_ops cn10k_hw_ops = { + .sq_aq_init = cn10k_sq_aq_init, + .sqe_flush = cn10k_sqe_flush, + .aura_freeptr = cn10k_aura_freeptr, + .refill_pool_ptrs = cn10k_refill_pool_ptrs, +}; + +int cn10k_pf_lmtst_init(struct otx2_nic *pf) +{ + int size, num_lines; + u64 base; + + if (!test_bit(CN10K_LMTST, &pf->hw.cap_flag)) { + pf->hw_ops = &otx2_hw_ops; + return 0; + } + + pf->hw_ops = &cn10k_hw_ops; + base = pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM) + + (MBOX_SIZE * (pf->total_vfs + 1)); + + size = pci_resource_len(pf->pdev, PCI_MBOX_BAR_NUM) - + (MBOX_SIZE * (pf->total_vfs + 1)); + + pf->hw.lmt_base = ioremap(base, size); + + if (!pf->hw.lmt_base) { + dev_err(pf->dev, "Unable to map PF LMTST region\n"); + return -ENOMEM; + } + + /* FIXME: Get the num of LMTST lines from LMT table */ + pf->tot_lmt_lines = size / LMT_LINE_SIZE; + num_lines = (pf->tot_lmt_lines - NIX_LMTID_BASE) / + pf->hw.tx_queues; + /* Number of LMT lines per SQ queues */ + pf->nix_lmt_lines = num_lines > 32 ? 32 : num_lines; + + pf->nix_lmt_size = pf->nix_lmt_lines * LMT_LINE_SIZE; + return 0; +} + +int cn10k_vf_lmtst_init(struct otx2_nic *vf) +{ + int size, num_lines; + + if (!test_bit(CN10K_LMTST, &vf->hw.cap_flag)) { + vf->hw_ops = &otx2_hw_ops; + return 0; + } + + vf->hw_ops = &cn10k_hw_ops; + size = pci_resource_len(vf->pdev, PCI_MBOX_BAR_NUM); + vf->hw.lmt_base = ioremap_wc(pci_resource_start(vf->pdev, + PCI_MBOX_BAR_NUM), + size); + if (!vf->hw.lmt_base) { + dev_err(vf->dev, "Unable to map VF LMTST region\n"); + return -ENOMEM; + } + + vf->tot_lmt_lines = size / LMT_LINE_SIZE; + /* LMTST lines per SQ */ + num_lines = (vf->tot_lmt_lines - NIX_LMTID_BASE) / + vf->hw.tx_queues; + vf->nix_lmt_lines = num_lines > 32 ? 32 : num_lines; + vf->nix_lmt_size = vf->nix_lmt_lines * LMT_LINE_SIZE; + return 0; +} +EXPORT_SYMBOL(cn10k_vf_lmtst_init); + +int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) +{ + struct nix_cn10k_aq_enq_req *aq; + struct otx2_nic *pfvf = dev; + struct otx2_snd_queue *sq; + + sq = &pfvf->qset.sq[qidx]; + sq->lmt_addr = (__force u64 *)((u64)pfvf->hw.nix_lmt_base + + (qidx * pfvf->nix_lmt_size)); + + /* Get memory to put this msg */ + aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); + if (!aq) + return -ENOMEM; + + aq->sq.cq = pfvf->hw.rx_queues + qidx; + aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ + aq->sq.cq_ena = 1; + aq->sq.ena = 1; + /* Only one SMQ is allocated, map all SQ's to that SMQ */ + aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; + /* FIXME: set based on NIX_AF_DWRR_RPM_MTU*/ + aq->sq.smq_rr_weight = OTX2_MAX_MTU; + aq->sq.default_chan = pfvf->hw.tx_chan_base; + aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ + aq->sq.sqb_aura = sqb_aura; + aq->sq.sq_int_ena = NIX_SQINT_BITS; + aq->sq.qint_idx = 0; + /* Due pipelining impact minimum 2000 unused SQ CQE's + * need to maintain to avoid CQ overflow. + */ + aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt)); + + /* Fill AQ info */ + aq->qidx = qidx; + aq->ctype = NIX_AQ_CTYPE_SQ; + aq->op = NIX_AQ_INSTOP_INIT; + + return otx2_sync_mbox_msg(&pfvf->mbox); +} + +#define NPA_MAX_BURST 16 +void cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) +{ + struct otx2_nic *pfvf = dev; + u64 ptrs[NPA_MAX_BURST]; + int num_ptrs = 1; + dma_addr_t bufptr; + + /* Refill pool with new buffers */ + while (cq->pool_ptrs) { + if (otx2_alloc_buffer(pfvf, cq, &bufptr)) { + if (num_ptrs--) + __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, + num_ptrs, + cq->rbpool->lmt_addr); + break; + } + cq->pool_ptrs--; + ptrs[num_ptrs] = (u64)bufptr + OTX2_HEAD_ROOM; + num_ptrs++; + if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) { + __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, + num_ptrs, + cq->rbpool->lmt_addr); + num_ptrs = 1; + } + } +} + +void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx) +{ + struct otx2_nic *pfvf = dev; + int lmt_id = NIX_LMTID_BASE + (qidx * pfvf->nix_lmt_lines); + u64 val = 0, tar_addr = 0; + + /* FIXME: val[0:10] LMT_ID. + * [12:15] no of LMTST - 1 in the burst. + * [19:63] data size of each LMTST in the burst except first. + */ + val = (lmt_id & 0x7FF); + /* Target address for LMTST flush tells HW how many 128bit + * words are present. + * tar_addr[6:4] size of first LMTST - 1 in units of 128b. + */ + tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4; + dma_wmb(); + memcpy(sq->lmt_addr, sq->sqe_base, size); + cn10k_lmt_flush(val, tar_addr); + + sq->head++; + sq->head &= (sq->sqe_cnt - 1); +} diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.h b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.h new file mode 100644 index 000000000000..e0bc595cbb78 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Marvell OcteonTx2 RVU Ethernet driver + * + * Copyright (C) 2020 Marvell. + */ + +#ifndef CN10K_H +#define CN10K_H + +#include "otx2_common.h" + +void cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq); +void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx); +int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura); +int cn10k_pf_lmtst_init(struct otx2_nic *pf); +int cn10k_vf_lmtst_init(struct otx2_nic *vf); +#endif /* CN10K_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index dbbdc3453f1a..2779802eed84 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -15,6 +15,7 @@ #include "otx2_reg.h" #include "otx2_common.h" #include "otx2_struct.h" +#include "cn10k.h" static void otx2_nix_rq_op_stats(struct queue_stats *stats, struct otx2_nic *pfvf, int qidx) @@ -526,6 +527,26 @@ static int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool, return ret; } +int otx2_alloc_buffer(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, + dma_addr_t *dma) +{ + if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma))) { + struct refill_work *work; + struct delayed_work *dwork; + + work = &pfvf->refill_wrk[cq->cq_idx]; + dwork = &work->pool_refill_work; + /* Schedule a task if no other task is running */ + if (!cq->refill_task_sched) { + cq->refill_task_sched = true; + schedule_delayed_work(dwork, + msecs_to_jiffies(100)); + } + return -ENOMEM; + } + return 0; +} + void otx2_tx_timeout(struct net_device *netdev, unsigned int txq) { struct otx2_nic *pfvf = netdev_priv(netdev); @@ -728,9 +749,6 @@ void otx2_sqb_flush(struct otx2_nic *pfvf) #define RQ_PASS_LVL_AURA (255 - ((95 * 256) / 100)) /* RED when 95% is full */ #define RQ_DROP_LVL_AURA (255 - ((99 * 256) / 100)) /* Drop when 99% is full */ -/* Send skid of 2000 packets required for CQ size of 4K CQEs. */ -#define SEND_CQ_SKID 2000 - static int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura) { struct otx2_qset *qset = &pfvf->qset; @@ -764,45 +782,14 @@ static int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura) return otx2_sync_mbox_msg(&pfvf->mbox); } -static int cn10k_sq_aq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) -{ - struct nix_cn10k_aq_enq_req *aq; - - /* Get memory to put this msg */ - aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); - if (!aq) - return -ENOMEM; - - aq->sq.cq = pfvf->hw.rx_queues + qidx; - aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ - aq->sq.cq_ena = 1; - aq->sq.ena = 1; - /* Only one SMQ is allocated, map all SQ's to that SMQ */ - aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; - /* FIXME: set based on NIX_AF_DWRR_RPM_MTU*/ - aq->sq.smq_rr_weight = OTX2_MAX_MTU; - aq->sq.default_chan = pfvf->hw.tx_chan_base; - aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ - aq->sq.sqb_aura = sqb_aura; - aq->sq.sq_int_ena = NIX_SQINT_BITS; - aq->sq.qint_idx = 0; - /* Due pipelining impact minimum 2000 unused SQ CQE's - * need to maintain to avoid CQ overflow. - */ - aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt)); - - /* Fill AQ info */ - aq->qidx = qidx; - aq->ctype = NIX_AQ_CTYPE_SQ; - aq->op = NIX_AQ_INSTOP_INIT; - - return otx2_sync_mbox_msg(&pfvf->mbox); -} - -static int otx2_sq_aq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) +int otx2_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) { + struct otx2_nic *pfvf = dev; + struct otx2_snd_queue *sq; struct nix_aq_enq_req *aq; + sq = &pfvf->qset.sq[qidx]; + sq->lmt_addr = (__force u64 *)(pfvf->reg_base + LMT_LF_LMTLINEX(qidx)); /* Get memory to put this msg */ aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); if (!aq) @@ -873,16 +860,12 @@ static int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) sq->sqe_thresh = ((sq->num_sqbs * sq->sqe_per_sqb) * 10) / 100; sq->aura_id = sqb_aura; sq->aura_fc_addr = pool->fc_addr->base; - sq->lmt_addr = (__force u64 *)(pfvf->reg_base + LMT_LF_LMTLINEX(qidx)); sq->io_addr = (__force u64)otx2_get_regaddr(pfvf, NIX_LF_OP_SENDX(0)); sq->stats.bytes = 0; sq->stats.pkts = 0; - if (is_dev_otx2(pfvf->pdev)) - return otx2_sq_aq_init(pfvf, qidx, sqb_aura); - else - return cn10k_sq_aq_init(pfvf, qidx, sqb_aura); + return pfvf->hw_ops->sq_aq_init(pfvf, qidx, sqb_aura); } @@ -987,7 +970,7 @@ static void otx2_pool_refill_task(struct work_struct *work) } return; } - otx2_aura_freeptr(pfvf, qidx, bufptr + OTX2_HEAD_ROOM); + pfvf->hw_ops->aura_freeptr(pfvf, qidx, bufptr + OTX2_HEAD_ROOM); cq->pool_ptrs--; } cq->refill_task_sched = false; @@ -1231,6 +1214,11 @@ static int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id, pool->rbsize = buf_size; + /* Set LMTST addr for NPA batch free */ + if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) + pool->lmt_addr = (__force u64 *)((u64)pfvf->hw.npa_lmt_base + + (pool_id * LMT_LINE_SIZE)); + /* Initialize this pool's context via AF */ aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); if (!aq) { @@ -1319,7 +1307,7 @@ int otx2_sq_aura_pool_init(struct otx2_nic *pfvf) for (ptr = 0; ptr < num_sqbs; ptr++) { if (otx2_alloc_rbuf(pfvf, pool, &bufptr)) return -ENOMEM; - otx2_aura_freeptr(pfvf, pool_id, bufptr); + pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr); sq->sqb_ptrs[sq->sqb_count++] = (u64)bufptr; } } @@ -1369,8 +1357,8 @@ int otx2_rq_aura_pool_init(struct otx2_nic *pfvf) for (ptr = 0; ptr < num_ptrs; ptr++) { if (otx2_alloc_rbuf(pfvf, pool, &bufptr)) return -ENOMEM; - otx2_aura_freeptr(pfvf, pool_id, - bufptr + OTX2_HEAD_ROOM); + pfvf->hw_ops->aura_freeptr(pfvf, pool_id, + bufptr + OTX2_HEAD_ROOM); } } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index 9ac9b420dd95..51aaa6ae0fd3 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -50,6 +50,9 @@ enum arua_mapped_qtypes { #define NIX_LF_ERR_VEC 0x81 #define NIX_LF_POISON_VEC 0x82 +/* Send skid of 2000 packets required for CQ size of 4K CQEs. */ +#define SEND_CQ_SKID 2000 + /* RSS configuration */ struct otx2_rss_ctx { u8 ind_tbl[MAX_RSS_INDIR_TBL_SIZE]; @@ -275,9 +278,18 @@ struct otx2_flow_config { struct list_head flow_list; }; +struct dev_hw_ops { + int (*sq_aq_init)(void *dev, u16 qidx, u16 sqb_aura); + void (*sqe_flush)(void *dev, struct otx2_snd_queue *sq, + int size, int qidx); + void (*refill_pool_ptrs)(void *dev, struct otx2_cq_queue *cq); + void (*aura_freeptr)(void *dev, int aura, u64 buf); +}; + struct otx2_nic { void __iomem *reg_base; struct net_device *netdev; + struct dev_hw_ops *hw_ops; void *iommu_domain; u16 max_frs; u16 rbsize; /* Receive buffer size */ @@ -507,10 +519,51 @@ static inline u64 otx2_atomic64_add(u64 incr, u64 *ptr) } #else -#define otx2_write128(lo, hi, addr) +#define otx2_write128(lo, hi, addr) writeq((hi) | (lo), addr) #define otx2_atomic64_add(incr, ptr) ({ *ptr += incr; }) #endif +static inline void __cn10k_aura_freeptr(struct otx2_nic *pfvf, u64 aura, + u64 *ptrs, u64 num_ptrs, + u64 *lmt_addr) +{ + u64 size = 0, count_eot = 0; + u64 tar_addr, val = 0; + + tar_addr = (__force u64)otx2_get_regaddr(pfvf, NPA_LF_AURA_BATCH_FREE0); + /* LMTID is same as AURA Id */ + val = (aura & 0x7FF) | BIT_ULL(63); + /* Set if [127:64] of last 128bit word has a valid pointer */ + count_eot = (num_ptrs % 2) ? 0ULL : 1ULL; + /* Set AURA ID to free pointer */ + ptrs[0] = (count_eot << 32) | (aura & 0xFFFFF); + /* Target address for LMTST flush tells HW how many 128bit + * words are valid from NPA_LF_AURA_BATCH_FREE0. + * + * tar_addr[6:4] is LMTST size-1 in units of 128b. + */ + if (num_ptrs > 2) { + size = (sizeof(u64) * num_ptrs) / 16; + if (!count_eot) + size++; + tar_addr |= ((size - 1) & 0x7) << 4; + } + memcpy(lmt_addr, ptrs, sizeof(u64) * num_ptrs); + /* Perform LMTST flush */ + cn10k_lmt_flush(val, tar_addr); +} + +static inline void cn10k_aura_freeptr(void *dev, int aura, u64 buf) +{ + struct otx2_nic *pfvf = dev; + struct otx2_pool *pool; + u64 ptrs[2]; + + pool = &pfvf->qset.pool[aura]; + ptrs[1] = buf; + __cn10k_aura_freeptr(pfvf, aura, ptrs, 2, pool->lmt_addr); +} + /* Alloc pointer from pool/aura */ static inline u64 otx2_aura_allocptr(struct otx2_nic *pfvf, int aura) { @@ -522,11 +575,12 @@ static inline u64 otx2_aura_allocptr(struct otx2_nic *pfvf, int aura) } /* Free pointer to a pool/aura */ -static inline void otx2_aura_freeptr(struct otx2_nic *pfvf, - int aura, u64 buf) +static inline void otx2_aura_freeptr(void *dev, int aura, u64 buf) { - otx2_write128(buf, (u64)aura | BIT_ULL(63), - otx2_get_regaddr(pfvf, NPA_LF_AURA_OP_FREE0)); + struct otx2_nic *pfvf = dev; + void __iomem *addr = otx2_get_regaddr(pfvf, NPA_LF_AURA_OP_FREE0); + + otx2_write128(buf, (u64)aura | BIT_ULL(63), addr); } static inline int otx2_get_pool_idx(struct otx2_nic *pfvf, int type, int idx) @@ -681,6 +735,10 @@ void otx2_ctx_disable(struct mbox *mbox, int type, bool npa); int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable); void otx2_cleanup_rx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq); void otx2_cleanup_tx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq); +int otx2_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura); +int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura); +int otx2_alloc_buffer(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, + dma_addr_t *dma); /* RSS configuration APIs*/ int otx2_rss_init(struct otx2_nic *pfvf); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index afafaec88c8b..f87cfcfc2832 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -22,6 +22,7 @@ #include "otx2_txrx.h" #include "otx2_struct.h" #include "otx2_ptp.h" +#include "cn10k.h" #include #define DRV_NAME "rvu_nicpf" @@ -46,39 +47,6 @@ enum { static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable); static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable); -static int cn10k_lmtst_init(struct otx2_nic *pf) -{ - int size, num_lines; - u64 base; - - if (!test_bit(CN10K_LMTST, &pf->hw.cap_flag)) - return 0; - - base = pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM) + - (MBOX_SIZE * (pf->total_vfs + 1)); - - size = pci_resource_len(pf->pdev, PCI_MBOX_BAR_NUM) - - (MBOX_SIZE * (pf->total_vfs + 1)); - - pf->hw.lmt_base = ioremap(base, size); - - if (!pf->hw.lmt_base) { - dev_err(pf->dev, "Unable to map PF LMTST region\n"); - return -ENOMEM; - } - - /* FIXME: Get the num of LMTST lines from LMT table */ - pf->tot_lmt_lines = size / LMT_LINE_SIZE; - num_lines = (pf->tot_lmt_lines - NIX_LMTID_BASE) / - pf->hw.tx_queues; - /* Number of LMT lines per SQ queues */ - pf->nix_lmt_lines = num_lines > 32 ? 32 : num_lines; - - pf->nix_lmt_size = pf->nix_lmt_lines * LMT_LINE_SIZE; - - return 0; -} - static int otx2_change_mtu(struct net_device *netdev, int new_mtu) { bool if_up = netif_running(netdev); @@ -2404,7 +2372,7 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_detach_rsrc; - err = cn10k_lmtst_init(pf); + err = cn10k_pf_lmtst_init(pf); if (err) goto err_detach_rsrc; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_reg.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_reg.h index 1e052d76a580..21b811c6ee0f 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_reg.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_reg.h @@ -94,6 +94,7 @@ #define NPA_LF_QINTX_INT_W1S(a) (NPA_LFBASE | 0x318 | (a) << 12) #define NPA_LF_QINTX_ENA_W1S(a) (NPA_LFBASE | 0x320 | (a) << 12) #define NPA_LF_QINTX_ENA_W1C(a) (NPA_LFBASE | 0x330 | (a) << 12) +#define NPA_LF_AURA_BATCH_FREE0 (NPA_LFBASE | 0x400) /* NIX LF registers */ #define NIX_LFBASE (BLKTYPE_NIX << RVU_FUNC_BLKADDR_SHIFT) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c index 68f80e75c8c7..59a7bd88d907 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c @@ -17,6 +17,7 @@ #include "otx2_struct.h" #include "otx2_txrx.h" #include "otx2_ptp.h" +#include "cn10k.h" #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx))) @@ -199,7 +200,8 @@ static void otx2_free_rcv_seg(struct otx2_nic *pfvf, struct nix_cqe_rx_s *cqe, sg = (struct nix_rx_sg_s *)start; seg_addr = &sg->seg_addr; for (seg = 0; seg < sg->segs; seg++, seg_addr++) - otx2_aura_freeptr(pfvf, qidx, *seg_addr & ~0x07ULL); + pfvf->hw_ops->aura_freeptr(pfvf, qidx, + *seg_addr & ~0x07ULL); start += sizeof(*sg); } } @@ -304,7 +306,6 @@ static int otx2_rx_napi_handler(struct otx2_nic *pfvf, { struct nix_cqe_rx_s *cqe; int processed_cqe = 0; - dma_addr_t bufptr; while (likely(processed_cqe < budget)) { cqe = (struct nix_cqe_rx_s *)CQE_ADDR(cq, cq->cq_head); @@ -330,28 +331,23 @@ static int otx2_rx_napi_handler(struct otx2_nic *pfvf, if (unlikely(!cq->pool_ptrs)) return 0; - /* Refill pool with new buffers */ + pfvf->hw_ops->refill_pool_ptrs(pfvf, cq); + + return processed_cqe; +} + +void otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) +{ + struct otx2_nic *pfvf = dev; + dma_addr_t bufptr; + while (cq->pool_ptrs) { - if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, &bufptr))) { - struct refill_work *work; - struct delayed_work *dwork; - - work = &pfvf->refill_wrk[cq->cq_idx]; - dwork = &work->pool_refill_work; - /* Schedule a task if no other task is running */ - if (!cq->refill_task_sched) { - cq->refill_task_sched = true; - schedule_delayed_work(dwork, - msecs_to_jiffies(100)); - } + if (otx2_alloc_buffer(pfvf, cq, &bufptr)) break; - } otx2_aura_freeptr(pfvf, cq->cq_idx, bufptr + OTX2_HEAD_ROOM); cq->pool_ptrs--; } - - return processed_cqe; } static int otx2_tx_napi_handler(struct otx2_nic *pfvf, @@ -438,7 +434,8 @@ int otx2_napi_handler(struct napi_struct *napi, int budget) return workdone; } -static void otx2_sqe_flush(struct otx2_snd_queue *sq, int size) +void otx2_sqe_flush(void *dev, struct otx2_snd_queue *sq, + int size, int qidx) { u64 status; @@ -796,7 +793,7 @@ static void otx2_sq_append_tso(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, sqe_hdr->sizem1 = (offset / 16) - 1; /* Flush SQE to HW */ - otx2_sqe_flush(sq, offset); + pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx); } } @@ -915,7 +912,7 @@ bool otx2_sq_append_skb(struct net_device *netdev, struct otx2_snd_queue *sq, netdev_tx_sent_queue(txq, skb->len); /* Flush SQE to HW */ - otx2_sqe_flush(sq, offset); + pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx); return true; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h index 73af15685657..d2b26b3357f3 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h @@ -114,6 +114,7 @@ struct otx2_cq_poll { struct otx2_pool { struct qmem *stack; struct qmem *fc_addr; + u64 *lmt_addr; u16 rbsize; }; @@ -156,4 +157,10 @@ static inline u64 otx2_iova_to_phys(void *iommu_domain, dma_addr_t dma_addr) int otx2_napi_handler(struct napi_struct *napi, int budget); bool otx2_sq_append_skb(struct net_device *netdev, struct otx2_snd_queue *sq, struct sk_buff *skb, u16 qidx); +void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, + int size, int qidx); +void otx2_sqe_flush(void *dev, struct otx2_snd_queue *sq, + int size, int qidx); +void otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq); +void cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq); #endif /* OTX2_TXRX_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c index 9ed850b75d59..31e03253e612 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c @@ -7,6 +7,7 @@ #include "otx2_common.h" #include "otx2_reg.h" +#include "cn10k.h" #define DRV_NAME "rvu_nicvf" #define DRV_STRING "Marvell RVU NIC Virtual Function Driver" @@ -27,31 +28,6 @@ enum { RVU_VF_INT_VEC_MBOX = 0x0, }; -static int cn10k_lmtst_init(struct otx2_nic *vf) -{ - int size, num_lines; - - if (!test_bit(CN10K_LMTST, &vf->hw.cap_flag)) - return 0; - - size = pci_resource_len(vf->pdev, PCI_MBOX_BAR_NUM); - vf->hw.lmt_base = ioremap_wc(pci_resource_start(vf->pdev, - PCI_MBOX_BAR_NUM), - size); - if (!vf->hw.lmt_base) { - dev_err(vf->dev, "Unable to map VF LMTST region\n"); - return -ENOMEM; - } - - vf->tot_lmt_lines = size / LMT_LINE_SIZE; - /* LMTST lines per SQ */ - num_lines = (vf->tot_lmt_lines - NIX_LMTID_BASE) / - vf->hw.tx_queues; - vf->nix_lmt_lines = num_lines > 32 ? 32 : num_lines; - vf->nix_lmt_size = vf->nix_lmt_lines * LMT_LINE_SIZE; - return 0; -} - static void otx2vf_process_vfaf_mbox_msg(struct otx2_nic *vf, struct mbox_msghdr *msg) { @@ -585,7 +561,7 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_detach_rsrc; - err = cn10k_lmtst_init(vf); + err = cn10k_vf_lmtst_init(vf); if (err) goto err_detach_rsrc; diff --git a/include/linux/soc/marvell/octeontx2/asm.h b/include/linux/soc/marvell/octeontx2/asm.h index ae2279fe830a..28c04d918f0f 100644 --- a/include/linux/soc/marvell/octeontx2/asm.h +++ b/include/linux/soc/marvell/octeontx2/asm.h @@ -22,8 +22,16 @@ : [rs]"r" (ioaddr)); \ (result); \ }) +#define cn10k_lmt_flush(val, addr) \ +({ \ + __asm__ volatile(".cpu generic+lse\n" \ + "steor %x[rf],[%[rs]]" \ + : [rf]"+r"(val) \ + : [rs]"r"(addr)); \ +}) #else #define otx2_lmt_flush(ioaddr) ({ 0; }) +#define cn10k_lmt_flush(val, addr) ({ addr = val; }) #endif #endif /* __SOC_OTX2_ASM_H */ -- cgit v1.2.3 From c5dbb89fc2ac013afe67b9e4fcb3743c02b567cd Mon Sep 17 00:00:00 2001 From: Florent Revest Date: Wed, 10 Feb 2021 12:14:03 +0100 Subject: bpf: Expose bpf_get_socket_cookie to tracing programs This needs a new helper that: - can work in a sleepable context (using sock_gen_cookie) - takes a struct sock pointer and checks that it's not NULL Signed-off-by: Florent Revest Signed-off-by: Alexei Starovoitov Acked-by: KP Singh Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20210210111406.785541-2-revest@chromium.org --- include/linux/bpf.h | 1 + include/uapi/linux/bpf.h | 8 ++++++++ kernel/trace/bpf_trace.c | 2 ++ net/core/filter.c | 12 ++++++++++++ tools/include/uapi/linux/bpf.h | 8 ++++++++ 5 files changed, 31 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 1c8ea682c007..cccaef1088ea 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1885,6 +1885,7 @@ extern const struct bpf_func_proto bpf_per_cpu_ptr_proto; extern const struct bpf_func_proto bpf_this_cpu_ptr_proto; extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto; extern const struct bpf_func_proto bpf_sock_from_file_proto; +extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto; const struct bpf_func_proto *bpf_tracing_func_proto( enum bpf_func_id func_id, const struct bpf_prog *prog); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index dbf10bf08582..07cc2e404291 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1673,6 +1673,14 @@ union bpf_attr { * Return * A 8-byte long unique number. * + * u64 bpf_get_socket_cookie(struct sock *sk) + * Description + * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts + * *sk*, but gets socket from a BTF **struct sock**. This helper + * also works for sleepable programs. + * Return + * A 8-byte long unique number or 0 if *sk* is NULL. + * * u32 bpf_get_socket_uid(struct sk_buff *skb) * Return * The owner UID of the socket associated to *skb*. If the socket diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 6c0018abe68a..845b2168e006 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -1760,6 +1760,8 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_sk_storage_delete_tracing_proto; case BPF_FUNC_sock_from_file: return &bpf_sock_from_file_proto; + case BPF_FUNC_get_socket_cookie: + return &bpf_get_socket_ptr_cookie_proto; #endif case BPF_FUNC_seq_printf: return prog->expected_attach_type == BPF_TRACE_ITER ? diff --git a/net/core/filter.c b/net/core/filter.c index e15d4741719a..57aaed478362 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -4631,6 +4631,18 @@ static const struct bpf_func_proto bpf_get_socket_cookie_sock_proto = { .arg1_type = ARG_PTR_TO_CTX, }; +BPF_CALL_1(bpf_get_socket_ptr_cookie, struct sock *, sk) +{ + return sk ? sock_gen_cookie(sk) : 0; +} + +const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto = { + .func = bpf_get_socket_ptr_cookie, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, +}; + BPF_CALL_1(bpf_get_socket_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx) { return __sock_gen_cookie(ctx->sk); diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index dbf10bf08582..07cc2e404291 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1673,6 +1673,14 @@ union bpf_attr { * Return * A 8-byte long unique number. * + * u64 bpf_get_socket_cookie(struct sock *sk) + * Description + * Equivalent to **bpf_get_socket_cookie**\ () helper that accepts + * *sk*, but gets socket from a BTF **struct sock**. This helper + * also works for sleepable programs. + * Return + * A 8-byte long unique number or 0 if *sk* is NULL. + * * u32 bpf_get_socket_uid(struct sk_buff *skb) * Return * The owner UID of the socket associated to *skb*. If the socket -- cgit v1.2.3 From 3b23a32a63219f51a5298bc55a65ecee866e79d0 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Thu, 11 Feb 2021 11:34:10 -0800 Subject: net: fix dev_ifsioc_locked() race condition dev_ifsioc_locked() is called with only RCU read lock, so when there is a parallel writer changing the mac address, it could get a partially updated mac address, as shown below: Thread 1 Thread 2 // eth_commit_mac_addr_change() memcpy(dev->dev_addr, addr->sa_data, ETH_ALEN); // dev_ifsioc_locked() memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,...); Close this race condition by guarding them with a RW semaphore, like netdev_get_name(). We can not use seqlock here as it does not allow blocking. The writers already take RTNL anyway, so this does not affect the slow path. To avoid bothering existing dev_set_mac_address() callers in drivers, introduce a new wrapper just for user-facing callers on ioctl and rtnetlink paths. Note, bonding also changes slave mac addresses but that requires a separate patch due to the complexity of bonding code. Fixes: 3710becf8a58 ("net: RCU locking for simple ioctl()") Reported-by: "Gong, Sishuai" Cc: Eric Dumazet Cc: Jakub Kicinski Signed-off-by: Cong Wang Signed-off-by: David S. Miller --- drivers/net/tap.c | 7 +++---- drivers/net/tun.c | 5 ++--- include/linux/netdevice.h | 3 +++ net/core/dev.c | 42 ++++++++++++++++++++++++++++++++++++++++++ net/core/dev_ioctl.c | 20 +++++++------------- net/core/rtnetlink.c | 2 +- 6 files changed, 58 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/tap.c b/drivers/net/tap.c index ff4aa35979a1..8e3a28ba6b28 100644 --- a/drivers/net/tap.c +++ b/drivers/net/tap.c @@ -1090,10 +1090,9 @@ static long tap_ioctl(struct file *file, unsigned int cmd, return -ENOLINK; } ret = 0; - u = tap->dev->type; + dev_get_mac_address(&sa, dev_net(tap->dev), tap->dev->name); if (copy_to_user(&ifr->ifr_name, tap->dev->name, IFNAMSIZ) || - copy_to_user(&ifr->ifr_hwaddr.sa_data, tap->dev->dev_addr, ETH_ALEN) || - put_user(u, &ifr->ifr_hwaddr.sa_family)) + copy_to_user(&ifr->ifr_hwaddr, &sa, sizeof(sa))) ret = -EFAULT; tap_put_tap_dev(tap); rtnl_unlock(); @@ -1108,7 +1107,7 @@ static long tap_ioctl(struct file *file, unsigned int cmd, rtnl_unlock(); return -ENOLINK; } - ret = dev_set_mac_address(tap->dev, &sa, NULL); + ret = dev_set_mac_address_user(tap->dev, &sa, NULL); tap_put_tap_dev(tap); rtnl_unlock(); return ret; diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 62690baa19bc..fc86da7f1628 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -3107,15 +3107,14 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd, case SIOCGIFHWADDR: /* Get hw address */ - memcpy(ifr.ifr_hwaddr.sa_data, tun->dev->dev_addr, ETH_ALEN); - ifr.ifr_hwaddr.sa_family = tun->dev->type; + dev_get_mac_address(&ifr.ifr_hwaddr, net, tun->dev->name); if (copy_to_user(argp, &ifr, ifreq_len)) ret = -EFAULT; break; case SIOCSIFHWADDR: /* Set hw address */ - ret = dev_set_mac_address(tun->dev, &ifr.ifr_hwaddr, NULL); + ret = dev_set_mac_address_user(tun->dev, &ifr.ifr_hwaddr, NULL); break; case TUNGETSNDBUF: diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index a20310ff5083..bfadf3b82f9c 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3902,6 +3902,9 @@ int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr, struct netlink_ext_ack *extack); int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa, struct netlink_ext_ack *extack); +int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa, + struct netlink_ext_ack *extack); +int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name); int dev_change_carrier(struct net_device *, bool new_carrier); int dev_get_phys_port_id(struct net_device *dev, struct netdev_phys_item_id *ppid); diff --git a/net/core/dev.c b/net/core/dev.c index 321d41a110e7..ce6291bc2e16 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -8937,6 +8937,48 @@ int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa, } EXPORT_SYMBOL(dev_set_mac_address); +static DECLARE_RWSEM(dev_addr_sem); + +int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa, + struct netlink_ext_ack *extack) +{ + int ret; + + down_write(&dev_addr_sem); + ret = dev_set_mac_address(dev, sa, extack); + up_write(&dev_addr_sem); + return ret; +} +EXPORT_SYMBOL(dev_set_mac_address_user); + +int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name) +{ + size_t size = sizeof(sa->sa_data); + struct net_device *dev; + int ret = 0; + + down_read(&dev_addr_sem); + rcu_read_lock(); + + dev = dev_get_by_name_rcu(net, dev_name); + if (!dev) { + ret = -ENODEV; + goto unlock; + } + if (!dev->addr_len) + memset(sa->sa_data, 0, size); + else + memcpy(sa->sa_data, dev->dev_addr, + min_t(size_t, size, dev->addr_len)); + sa->sa_family = dev->type; + +unlock: + rcu_read_unlock(); + up_read(&dev_addr_sem); + return ret; +} +EXPORT_SYMBOL(dev_get_mac_address); + /** * dev_change_carrier - Change device carrier * @dev: device diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index db8a0ff86f36..478d032f34ac 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -123,17 +123,6 @@ static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cm ifr->ifr_mtu = dev->mtu; return 0; - case SIOCGIFHWADDR: - if (!dev->addr_len) - memset(ifr->ifr_hwaddr.sa_data, 0, - sizeof(ifr->ifr_hwaddr.sa_data)); - else - memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr, - min(sizeof(ifr->ifr_hwaddr.sa_data), - (size_t)dev->addr_len)); - ifr->ifr_hwaddr.sa_family = dev->type; - return 0; - case SIOCGIFSLAVE: err = -EINVAL; break; @@ -274,7 +263,7 @@ static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd) case SIOCSIFHWADDR: if (dev->addr_len > sizeof(struct sockaddr)) return -EINVAL; - return dev_set_mac_address(dev, &ifr->ifr_hwaddr, NULL); + return dev_set_mac_address_user(dev, &ifr->ifr_hwaddr, NULL); case SIOCSIFHWBROADCAST: if (ifr->ifr_hwaddr.sa_family != dev->type) @@ -418,6 +407,12 @@ int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr, bool *need_c */ switch (cmd) { + case SIOCGIFHWADDR: + dev_load(net, ifr->ifr_name); + ret = dev_get_mac_address(&ifr->ifr_hwaddr, net, ifr->ifr_name); + if (colon) + *colon = ':'; + return ret; /* * These ioctl calls: * - can be done by all. @@ -427,7 +422,6 @@ int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr, bool *need_c case SIOCGIFFLAGS: case SIOCGIFMETRIC: case SIOCGIFMTU: - case SIOCGIFHWADDR: case SIOCGIFSLAVE: case SIOCGIFMAP: case SIOCGIFINDEX: diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index c313aaf2bce1..0edc0b2baaa4 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -2660,7 +2660,7 @@ static int do_setlink(const struct sk_buff *skb, sa->sa_family = dev->type; memcpy(sa->sa_data, nla_data(tb[IFLA_ADDRESS]), dev->addr_len); - err = dev_set_mac_address(dev, sa, extack); + err = dev_set_mac_address_user(dev, sa, extack); kfree(sa); if (err) goto errout; -- cgit v1.2.3 From 38f3885edbef8a77b25c4d13f3de06a7b93d02de Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 12 Feb 2021 16:11:17 +0200 Subject: ACPI: property: Remove dead code After the commit 3a7a2ab839ad couple of functions became a dead code. Moreover, for all these years nobody used them. Remove. Fixes: 3a7a2ab839ad ("ACPI / property: Extend fwnode_property_* to data-only subnodes") Signed-off-by: Andy Shevchenko Signed-off-by: Rafael J. Wysocki --- drivers/acpi/property.c | 20 -------------------- include/linux/acpi.h | 21 --------------------- 2 files changed, 41 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c index 16b28084c1ca..22ccab4e7c6d 100644 --- a/drivers/acpi/property.c +++ b/drivers/acpi/property.c @@ -841,20 +841,6 @@ static int acpi_data_prop_read_single(const struct acpi_device_data *data, return ret; } -int acpi_dev_prop_read_single(struct acpi_device *adev, const char *propname, - enum dev_prop_type proptype, void *val) -{ - int ret; - - if (!adev || !val) - return -EINVAL; - - ret = acpi_data_prop_read_single(&adev->data, propname, proptype, val); - if (ret < 0 || proptype != ACPI_TYPE_STRING) - return ret; - return 0; -} - static int acpi_copy_property_array_u8(const union acpi_object *items, u8 *val, size_t nval) { @@ -995,12 +981,6 @@ static int acpi_data_prop_read(const struct acpi_device_data *data, return ret; } -int acpi_dev_prop_read(const struct acpi_device *adev, const char *propname, - enum dev_prop_type proptype, void *val, size_t nval) -{ - return adev ? acpi_data_prop_read(&adev->data, propname, proptype, val, nval) : -EINVAL; -} - /** * acpi_node_prop_read - retrieve the value of an ACPI property with given name. * @fwnode: Firmware node to get the property from. diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 053bf05fb1f7..a47eaf448131 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1114,14 +1114,9 @@ acpi_data_add_props(struct acpi_device_data *data, const guid_t *guid, int acpi_node_prop_get(const struct fwnode_handle *fwnode, const char *propname, void **valptr); -int acpi_dev_prop_read_single(struct acpi_device *adev, - const char *propname, enum dev_prop_type proptype, - void *val); int acpi_node_prop_read(const struct fwnode_handle *fwnode, const char *propname, enum dev_prop_type proptype, void *val, size_t nval); -int acpi_dev_prop_read(const struct acpi_device *adev, const char *propname, - enum dev_prop_type proptype, void *val, size_t nval); struct fwnode_handle *acpi_get_next_subnode(const struct fwnode_handle *fwnode, struct fwnode_handle *child); @@ -1223,14 +1218,6 @@ static inline int acpi_node_prop_get(const struct fwnode_handle *fwnode, return -ENXIO; } -static inline int acpi_dev_prop_read_single(const struct acpi_device *adev, - const char *propname, - enum dev_prop_type proptype, - void *val) -{ - return -ENXIO; -} - static inline int acpi_node_prop_read(const struct fwnode_handle *fwnode, const char *propname, enum dev_prop_type proptype, @@ -1239,14 +1226,6 @@ static inline int acpi_node_prop_read(const struct fwnode_handle *fwnode, return -ENXIO; } -static inline int acpi_dev_prop_read(const struct acpi_device *adev, - const char *propname, - enum dev_prop_type proptype, - void *val, size_t nval) -{ - return -ENXIO; -} - static inline struct fwnode_handle * acpi_get_next_subnode(const struct fwnode_handle *fwnode, struct fwnode_handle *child) -- cgit v1.2.3 From 325aa816143228a0b3472074ffb50d55ac3f04fe Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 12 Feb 2021 16:11:18 +0200 Subject: ACPI: property: Make acpi_node_prop_read() static There is no users outside of property.c. No need to export acpi_node_prop_read(), hence make it static. Fixes: 3708184afc77 ("device property: Move FW type specific functionality to FW specific files") Signed-off-by: Andy Shevchenko Signed-off-by: Rafael J. Wysocki --- drivers/acpi/property.c | 6 +++--- include/linux/acpi.h | 11 ----------- 2 files changed, 3 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c index 22ccab4e7c6d..2b65ad9b4c0d 100644 --- a/drivers/acpi/property.c +++ b/drivers/acpi/property.c @@ -993,9 +993,9 @@ static int acpi_data_prop_read(const struct acpi_device_data *data, * of the property. Otherwise, read at most @nval values to the array at the * location pointed to by @val. */ -int acpi_node_prop_read(const struct fwnode_handle *fwnode, - const char *propname, enum dev_prop_type proptype, - void *val, size_t nval) +static int acpi_node_prop_read(const struct fwnode_handle *fwnode, + const char *propname, enum dev_prop_type proptype, + void *val, size_t nval) { return acpi_data_prop_read(acpi_device_data_of_node(fwnode), propname, proptype, val, nval); diff --git a/include/linux/acpi.h b/include/linux/acpi.h index a47eaf448131..dc6e1f39dc6f 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1114,9 +1114,6 @@ acpi_data_add_props(struct acpi_device_data *data, const guid_t *guid, int acpi_node_prop_get(const struct fwnode_handle *fwnode, const char *propname, void **valptr); -int acpi_node_prop_read(const struct fwnode_handle *fwnode, - const char *propname, enum dev_prop_type proptype, - void *val, size_t nval); struct fwnode_handle *acpi_get_next_subnode(const struct fwnode_handle *fwnode, struct fwnode_handle *child); @@ -1218,14 +1215,6 @@ static inline int acpi_node_prop_get(const struct fwnode_handle *fwnode, return -ENXIO; } -static inline int acpi_node_prop_read(const struct fwnode_handle *fwnode, - const char *propname, - enum dev_prop_type proptype, - void *val, size_t nval) -{ - return -ENXIO; -} - static inline struct fwnode_handle * acpi_get_next_subnode(const struct fwnode_handle *fwnode, struct fwnode_handle *child) -- cgit v1.2.3 From 5f7d57280c1982d993d5f4ff0edac310f820f607 Mon Sep 17 00:00:00 2001 From: Jesper Dangaard Brouer Date: Tue, 9 Feb 2021 14:38:29 +0100 Subject: bpf: Drop MTU check when doing TC-BPF redirect to ingress The use-case for dropping the MTU check when TC-BPF does redirect to ingress, is described by Eyal Birger in email[0]. The summary is the ability to increase packet size (e.g. with IPv6 headers for NAT64) and ingress redirect packet and let normal netstack fragment packet as needed. [0] https://lore.kernel.org/netdev/CAHsH6Gug-hsLGHQ6N0wtixdOa85LDZ3HNRHVd0opR=19Qo4W4Q@mail.gmail.com/ V15: - missing static for function declaration V9: - Make net_device "up" (IFF_UP) check explicit in skb_do_redirect V4: - Keep net_device "up" (IFF_UP) check. - Adjustment to handle bpf_redirect_peer() helper Signed-off-by: Jesper Dangaard Brouer Signed-off-by: Daniel Borkmann Acked-by: John Fastabend Link: https://lore.kernel.org/bpf/161287790971.790810.11785274340154740591.stgit@firesoul --- include/linux/netdevice.h | 32 ++++++++++++++++++++++++++++++-- net/core/dev.c | 32 ++++++++++++++------------------ net/core/filter.c | 6 +++--- 3 files changed, 47 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index ef517254367d..b9bcbfde7849 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -3931,14 +3931,42 @@ int xdp_umem_query(struct net_device *dev, u16 queue_id); int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb); int dev_forward_skb(struct net_device *dev, struct sk_buff *skb); +int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb); bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb); +static __always_inline bool __is_skb_forwardable(const struct net_device *dev, + const struct sk_buff *skb, + const bool check_mtu) +{ + const u32 vlan_hdr_len = 4; /* VLAN_HLEN */ + unsigned int len; + + if (!(dev->flags & IFF_UP)) + return false; + + if (!check_mtu) + return true; + + len = dev->mtu + dev->hard_header_len + vlan_hdr_len; + if (skb->len <= len) + return true; + + /* if TSO is enabled, we don't care about the length as the packet + * could be forwarded without being segmented before + */ + if (skb_is_gso(skb)) + return true; + + return false; +} + static __always_inline int ____dev_forward_skb(struct net_device *dev, - struct sk_buff *skb) + struct sk_buff *skb, + const bool check_mtu) { if (skb_orphan_frags(skb, GFP_ATOMIC) || - unlikely(!is_skb_forwardable(dev, skb))) { + unlikely(!__is_skb_forwardable(dev, skb, check_mtu))) { atomic_long_inc(&dev->rx_dropped); kfree_skb(skb); return NET_RX_DROP; diff --git a/net/core/dev.c b/net/core/dev.c index 07a0347c33fb..8c820fe0eff3 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2194,28 +2194,14 @@ static inline void net_timestamp_set(struct sk_buff *skb) bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb) { - unsigned int len; - - if (!(dev->flags & IFF_UP)) - return false; - - len = dev->mtu + dev->hard_header_len + VLAN_HLEN; - if (skb->len <= len) - return true; - - /* if TSO is enabled, we don't care about the length as the packet - * could be forwarded without being segmented before - */ - if (skb_is_gso(skb)) - return true; - - return false; + return __is_skb_forwardable(dev, skb, true); } EXPORT_SYMBOL_GPL(is_skb_forwardable); -int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) +static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb, + bool check_mtu) { - int ret = ____dev_forward_skb(dev, skb); + int ret = ____dev_forward_skb(dev, skb, check_mtu); if (likely(!ret)) { skb->protocol = eth_type_trans(skb, dev); @@ -2224,6 +2210,11 @@ int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) return ret; } + +int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) +{ + return __dev_forward_skb2(dev, skb, true); +} EXPORT_SYMBOL_GPL(__dev_forward_skb); /** @@ -2250,6 +2241,11 @@ int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) } EXPORT_SYMBOL_GPL(dev_forward_skb); +int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb) +{ + return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb); +} + static inline int deliver_skb(struct sk_buff *skb, struct packet_type *pt_prev, struct net_device *orig_dev) diff --git a/net/core/filter.c b/net/core/filter.c index 439f43f00483..7059cf604d94 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -2083,13 +2083,13 @@ static const struct bpf_func_proto bpf_csum_level_proto = { static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb) { - return dev_forward_skb(dev, skb); + return dev_forward_skb_nomtu(dev, skb); } static inline int __bpf_rx_skb_no_mac(struct net_device *dev, struct sk_buff *skb) { - int ret = ____dev_forward_skb(dev, skb); + int ret = ____dev_forward_skb(dev, skb, false); if (likely(!ret)) { skb->dev = dev; @@ -2480,7 +2480,7 @@ int skb_do_redirect(struct sk_buff *skb) goto out_drop; dev = ops->ndo_get_peer_dev(dev); if (unlikely(!dev || - !is_skb_forwardable(dev, skb) || + !(dev->flags & IFF_UP) || net_eq(net, dev_net(dev)))) goto out_drop; skb->dev = dev; -- cgit v1.2.3 From e5069b9c23b3857db986c58801bebe450cff3392 Mon Sep 17 00:00:00 2001 From: Dmitrii Banshchikov Date: Sat, 13 Feb 2021 00:56:41 +0400 Subject: bpf: Support pointers in global func args Add an ability to pass a pointer to a type with known size in arguments of a global function. Such pointers may be used to overcome the limit on the maximum number of arguments, avoid expensive and tricky workarounds and to have multiple output arguments. A referenced type may contain pointers but indirect access through them isn't supported. The implementation consists of two parts. If a global function has an argument that is a pointer to a type with known size then: 1) In btf_check_func_arg_match(): check that the corresponding register points to NULL or to a valid memory region that is large enough to contain the expected argument's type. 2) In btf_prepare_func_args(): set the corresponding register type to PTR_TO_MEM_OR_NULL and its size to the size of the expected type. Only global functions are supported because allowance of pointers for static functions might break validation. Consider the following scenario. A static function has a pointer argument. A caller passes pointer to its stack memory. Because the callee can change referenced memory verifier cannot longer assume any particular slot type of the caller's stack memory hence the slot type is changed to SLOT_MISC. If there is an operation that relies on slot type other than SLOT_MISC then verifier won't be able to infer safety of the operation. When verifier sees a static function that has a pointer argument different from PTR_TO_CTX then it skips arguments check and continues with "inline" validation with more information available. The operation that relies on the particular slot type now succeeds. Because global functions were not allowed to have pointer arguments different from PTR_TO_CTX it's not possible to break existing and valid code. Signed-off-by: Dmitrii Banshchikov Signed-off-by: Alexei Starovoitov Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20210212205642.620788-4-me@ubique.spb.ru --- include/linux/bpf_verifier.h | 2 ++ kernel/bpf/btf.c | 55 ++++++++++++++++++++++++++++++++++++-------- kernel/bpf/verifier.c | 30 ++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 532c97836d0d..971b33aca13d 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -471,6 +471,8 @@ bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt); int check_ctx_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int regno); +int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + u32 regno, u32 mem_size); /* this lives here instead of in bpf.h because it needs to dereference tgt_prog */ static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog, diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index bd5d2c563693..2efeb5f4b343 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5297,9 +5297,10 @@ int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_prog *prog = env->prog; struct btf *btf = prog->aux->btf; const struct btf_param *args; - const struct btf_type *t; - u32 i, nargs, btf_id; + const struct btf_type *t, *ref_t; + u32 i, nargs, btf_id, type_size; const char *tname; + bool is_global; if (!prog->aux->func_info) return -EINVAL; @@ -5333,6 +5334,8 @@ int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, bpf_log(log, "Function %s has %d > 5 args\n", tname, nargs); goto out; } + + is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; /* check that BTF function arguments match actual types that the * verifier sees. */ @@ -5349,10 +5352,6 @@ int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, goto out; } if (btf_type_is_ptr(t)) { - if (reg->type == SCALAR_VALUE) { - bpf_log(log, "R%d is not a pointer\n", i + 1); - goto out; - } /* If function expects ctx type in BTF check that caller * is passing PTR_TO_CTX. */ @@ -5367,6 +5366,25 @@ int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, goto out; continue; } + + if (!is_global) + goto out; + + t = btf_type_skip_modifiers(btf, t->type, NULL); + + ref_t = btf_resolve_size(btf, t, &type_size); + if (IS_ERR(ref_t)) { + bpf_log(log, + "arg#%d reference type('%s %s') size cannot be determined: %ld\n", + i, btf_type_str(t), btf_name_by_offset(btf, t->name_off), + PTR_ERR(ref_t)); + goto out; + } + + if (check_mem_reg(env, reg, i + 1, type_size)) + goto out; + + continue; } bpf_log(log, "Unrecognized arg#%d type %s\n", i, btf_kind_str[BTF_INFO_KIND(t->info)]); @@ -5397,7 +5415,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, enum bpf_prog_type prog_type = prog->type; struct btf *btf = prog->aux->btf; const struct btf_param *args; - const struct btf_type *t; + const struct btf_type *t, *ref_t; u32 i, nargs, btf_id; const char *tname; @@ -5470,9 +5488,26 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, reg->type = SCALAR_VALUE; continue; } - if (btf_type_is_ptr(t) && - btf_get_prog_ctx_type(log, btf, t, prog_type, i)) { - reg->type = PTR_TO_CTX; + if (btf_type_is_ptr(t)) { + if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) { + reg->type = PTR_TO_CTX; + continue; + } + + t = btf_type_skip_modifiers(btf, t->type, NULL); + + ref_t = btf_resolve_size(btf, t, ®->mem_size); + if (IS_ERR(ref_t)) { + bpf_log(log, + "arg#%d reference type('%s %s') size cannot be determined: %ld\n", + i, btf_type_str(t), btf_name_by_offset(btf, t->name_off), + PTR_ERR(ref_t)); + return -EINVAL; + } + + reg->type = PTR_TO_MEM_OR_NULL; + reg->id = ++env->id_gen; + continue; } bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n", diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a9f75bd7f8d3..11a242932a2c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4272,6 +4272,29 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, } } +int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + u32 regno, u32 mem_size) +{ + if (register_is_null(reg)) + return 0; + + if (reg_type_may_be_null(reg->type)) { + /* Assuming that the register contains a value check if the memory + * access is safe. Temporarily save and restore the register's state as + * the conversion shouldn't be visible to a caller. + */ + const struct bpf_reg_state saved_reg = *reg; + int rv; + + mark_ptr_not_null_reg(reg); + rv = check_helper_mem_access(env, regno, mem_size, true, NULL); + *reg = saved_reg; + return rv; + } + + return check_helper_mem_access(env, regno, mem_size, true, NULL); +} + /* Implementation details: * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL * Two bpf_map_lookups (even with the same key) will have different reg->id. @@ -11960,6 +11983,13 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) mark_reg_known_zero(env, regs, i); else if (regs[i].type == SCALAR_VALUE) mark_reg_unknown(env, regs, i); + else if (regs[i].type == PTR_TO_MEM_OR_NULL) { + const u32 mem_size = regs[i].mem_size; + + mark_reg_known_zero(env, regs, i); + regs[i].mem_size = mem_size; + regs[i].id = ++env->id_gen; + } } } else { /* 1st arg to a function */ -- cgit v1.2.3 From 6d4e9a8efe3d59f31367d79e970c2f328da139a4 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 10 Feb 2021 10:56:39 +0100 Subject: driver core: lift dma_default_coherent into common code Lift the dma_default_coherent variable from the mips architecture code to the driver core. This allows an architecture to sdefault all device to be DMA coherent at run time, even if the kernel is build with support for DMA noncoherent device. By allowing device_initialize to set the ->dma_coherent field to this default the amount of arch hooks required for this behavior can be greatly reduced. Signed-off-by: Christoph Hellwig Acked-by: Greg Kroah-Hartman Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kconfig | 1 - arch/mips/alchemy/common/setup.c | 2 +- arch/mips/include/asm/dma-coherence.h | 22 ---------------------- arch/mips/kernel/setup.c | 4 ---- arch/mips/mm/c-r4k.c | 2 +- arch/mips/mm/dma-noncoherent.c | 1 - arch/mips/mti-malta/malta-setup.c | 2 +- arch/mips/pci/pci-alchemy.c | 2 +- arch/mips/pistachio/init.c | 1 - drivers/base/core.c | 6 ++++++ include/linux/dma-map-ops.h | 5 ++--- kernel/dma/Kconfig | 3 --- kernel/dma/mapping.c | 2 ++ 13 files changed, 14 insertions(+), 39 deletions(-) delete mode 100644 arch/mips/include/asm/dma-coherence.h (limited to 'include/linux') diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 57b06798706c..a6d73c763be1 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1163,7 +1163,6 @@ config ARCH_SUPPORTS_UPROBES bool config DMA_MAYBE_COHERENT - select ARCH_HAS_DMA_COHERENCE_H select DMA_NONCOHERENT bool diff --git a/arch/mips/alchemy/common/setup.c b/arch/mips/alchemy/common/setup.c index 39e5b9cd882b..2388d68786f4 100644 --- a/arch/mips/alchemy/common/setup.c +++ b/arch/mips/alchemy/common/setup.c @@ -28,8 +28,8 @@ #include #include #include +#include /* for dma_default_coherent */ -#include #include #include diff --git a/arch/mips/include/asm/dma-coherence.h b/arch/mips/include/asm/dma-coherence.h deleted file mode 100644 index 846c5ade30d1..000000000000 --- a/arch/mips/include/asm/dma-coherence.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2006 Ralf Baechle - * - */ -#ifndef __ASM_DMA_COHERENCE_H -#define __ASM_DMA_COHERENCE_H - -#ifdef CONFIG_DMA_MAYBE_COHERENT -extern bool dma_default_coherent; -static inline bool dev_is_dma_coherent(struct device *dev) -{ - return dma_default_coherent; -} -#else -#define dma_default_coherent (!IS_ENABLED(CONFIG_DMA_NONCOHERENT)) -#endif - -#endif diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 6008f45ad081..f0f533294311 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include @@ -803,9 +802,6 @@ arch_initcall(debugfs_mips); #endif #ifdef CONFIG_DMA_MAYBE_COHERENT -bool dma_default_coherent; -EXPORT_SYMBOL_GPL(dma_default_coherent); - static int __init setcoherentio(char *str) { dma_default_coherent = true; diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index bbfab94194b9..74b09e801c3a 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -19,6 +19,7 @@ #include #include #include +#include /* for dma_default_coherent */ #include #include @@ -35,7 +36,6 @@ #include #include /* for run_uncached() */ #include -#include #include /* diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c index 38d3d9143b47..90b562753eb8 100644 --- a/arch/mips/mm/dma-noncoherent.c +++ b/arch/mips/mm/dma-noncoherent.c @@ -10,7 +10,6 @@ #include #include -#include #include /* diff --git a/arch/mips/mti-malta/malta-setup.c b/arch/mips/mti-malta/malta-setup.c index 1cdcb76d393e..21cb3ac1237b 100644 --- a/arch/mips/mti-malta/malta-setup.c +++ b/arch/mips/mti-malta/malta-setup.c @@ -13,8 +13,8 @@ #include #include #include +#include /* for dma_default_coherent */ -#include #include #include #include diff --git a/arch/mips/pci/pci-alchemy.c b/arch/mips/pci/pci-alchemy.c index 54c86b40d304..1c722dd0c130 100644 --- a/arch/mips/pci/pci-alchemy.c +++ b/arch/mips/pci/pci-alchemy.c @@ -17,8 +17,8 @@ #include #include #include +#include /* for dma_default_coherent */ -#include #include #include diff --git a/arch/mips/pistachio/init.c b/arch/mips/pistachio/init.c index 8e83262e9da8..e0bacfc3c6b4 100644 --- a/arch/mips/pistachio/init.c +++ b/arch/mips/pistachio/init.c @@ -13,7 +13,6 @@ #include #include -#include #include #include #include diff --git a/drivers/base/core.c b/drivers/base/core.c index 25e08e5f40bd..fb2988f955cb 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -28,6 +28,7 @@ #include #include #include +#include /* for dma_default_coherent */ #include "base.h" #include "power/power.h" @@ -2585,6 +2586,11 @@ void device_initialize(struct device *dev) INIT_LIST_HEAD(&dev->links.suppliers); INIT_LIST_HEAD(&dev->links.defer_sync); dev->links.status = DL_DEV_NO_DRIVER; +#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ + defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) + dev->dma_coherent = dma_default_coherent; +#endif } EXPORT_SYMBOL_GPL(device_initialize); diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 70fcd0f610ea..1e98b8c1e055 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -229,11 +229,10 @@ bool dma_free_from_pool(struct device *dev, void *start, size_t size); int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start, dma_addr_t dma_start, u64 size); -#ifdef CONFIG_ARCH_HAS_DMA_COHERENCE_H -#include -#elif defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ +#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) +extern bool dma_default_coherent; static inline bool dev_is_dma_coherent(struct device *dev) { return dev->dma_coherent; diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig index 479fc145acfc..77b405508743 100644 --- a/kernel/dma/Kconfig +++ b/kernel/dma/Kconfig @@ -33,9 +33,6 @@ config NEED_DMA_MAP_STATE config ARCH_DMA_ADDR_T_64BIT def_bool 64BIT || PHYS_ADDR_T_64BIT -config ARCH_HAS_DMA_COHERENCE_H - bool - config ARCH_HAS_DMA_SET_MASK bool diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index f87a89d08654..84de6b1c5fab 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -16,6 +16,8 @@ #include "debug.h" #include "direct.h" +bool dma_default_coherent; + /* * Managed DMA API */ -- cgit v1.2.3 From fec6e49b63989657bc4076dad99fa51d5ece34da Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Sat, 13 Feb 2021 14:12:02 +0000 Subject: skbuff: remove __kfree_skb_flush() This function isn't much needed as NAPI skb queue gets bulk-freed anyway when there's no more room, and even may reduce the efficiency of bulk operations. It will be even less needed after reusing skb cache on allocation path, so remove it and this way lighten network softirqs a bit. Suggested-by: Eric Dumazet Signed-off-by: Alexander Lobakin Signed-off-by: David S. Miller --- include/linux/skbuff.h | 1 - net/core/dev.c | 7 +------ net/core/skbuff.c | 12 ------------ 3 files changed, 1 insertion(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 0a4e91a2f873..0e0707296098 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2919,7 +2919,6 @@ static inline struct sk_buff *napi_alloc_skb(struct napi_struct *napi, } void napi_consume_skb(struct sk_buff *skb, int budget); -void __kfree_skb_flush(void); void __kfree_skb_defer(struct sk_buff *skb); /** diff --git a/net/core/dev.c b/net/core/dev.c index ce6291bc2e16..631807c196ad 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4944,8 +4944,6 @@ static __latent_entropy void net_tx_action(struct softirq_action *h) else __kfree_skb_defer(skb); } - - __kfree_skb_flush(); } if (sd->output_queue) { @@ -7012,7 +7010,6 @@ static int napi_threaded_poll(void *data) __napi_poll(napi, &repoll); netpoll_poll_unlock(have); - __kfree_skb_flush(); local_bh_enable(); if (!repoll) @@ -7042,7 +7039,7 @@ static __latent_entropy void net_rx_action(struct softirq_action *h) if (list_empty(&list)) { if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) - goto out; + return; break; } @@ -7069,8 +7066,6 @@ static __latent_entropy void net_rx_action(struct softirq_action *h) __raise_softirq_irqoff(NET_RX_SOFTIRQ); net_rps_action_and_irq_enable(sd); -out: - __kfree_skb_flush(); } struct netdev_adjacent { diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 1c6f6ef70339..4be2bb969535 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -838,18 +838,6 @@ void __consume_stateless_skb(struct sk_buff *skb) kfree_skbmem(skb); } -void __kfree_skb_flush(void) -{ - struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); - - /* flush skb_cache if containing objects */ - if (nc->skb_count) { - kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count, - nc->skb_cache); - nc->skb_count = 0; - } -} - static inline void _kfree_skb_defer(struct sk_buff *skb) { struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); -- cgit v1.2.3 From f450d539c05a14c103dd174718f81bb2fe65cb4b Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Sat, 13 Feb 2021 14:12:25 +0000 Subject: skbuff: introduce {,__}napi_build_skb() which reuses NAPI cache heads Instead of just bulk-flushing skbuff_heads queued up through napi_consume_skb() or __kfree_skb_defer(), try to reuse them on allocation path. If the cache is empty on allocation, bulk-allocate the first 16 elements, which is more efficient than per-skb allocation. If the cache is full on freeing, bulk-wipe the second half of the cache (32 elements). This also includes custom KASAN poisoning/unpoisoning to be double sure there are no use-after-free cases. To not change current behaviour, introduce a new function, napi_build_skb(), to optionally use a new approach later in drivers. Note on selected bulk size, 16: - this equals to XDP_BULK_QUEUE_SIZE, DEV_MAP_BULK_SIZE and especially VETH_XDP_BATCH, which is also used to bulk-allocate skbuff_heads and was tested on powerful setups; - this also showed the best performance in the actual test series (from the array of {8, 16, 32}). Suggested-by: Edward Cree # Divide on two halves Suggested-by: Eric Dumazet # KASAN poisoning Cc: Dmitry Vyukov # Help with KASAN Cc: Paolo Abeni # Reduced batch size Signed-off-by: Alexander Lobakin Signed-off-by: David S. Miller --- include/linux/skbuff.h | 2 ++ net/core/skbuff.c | 94 +++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 83 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 0e0707296098..906122eac82a 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1087,6 +1087,8 @@ struct sk_buff *build_skb(void *data, unsigned int frag_size); struct sk_buff *build_skb_around(struct sk_buff *skb, void *data, unsigned int frag_size); +struct sk_buff *napi_build_skb(void *data, unsigned int frag_size); + /** * alloc_skb - allocate a network buffer * @size: size to allocate diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 860a9d4f752f..9e1a8ded4acc 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -120,6 +120,8 @@ static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) } #define NAPI_SKB_CACHE_SIZE 64 +#define NAPI_SKB_CACHE_BULK 16 +#define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2) struct napi_alloc_cache { struct page_frag_cache page; @@ -164,6 +166,25 @@ void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) } EXPORT_SYMBOL(__netdev_alloc_frag_align); +static struct sk_buff *napi_skb_cache_get(void) +{ + struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); + struct sk_buff *skb; + + if (unlikely(!nc->skb_count)) + nc->skb_count = kmem_cache_alloc_bulk(skbuff_head_cache, + GFP_ATOMIC, + NAPI_SKB_CACHE_BULK, + nc->skb_cache); + if (unlikely(!nc->skb_count)) + return NULL; + + skb = nc->skb_cache[--nc->skb_count]; + kasan_unpoison_object_data(skbuff_head_cache, skb); + + return skb; +} + /* Caller must provide SKB that is memset cleared */ static void __build_skb_around(struct sk_buff *skb, void *data, unsigned int frag_size) @@ -265,6 +286,53 @@ struct sk_buff *build_skb_around(struct sk_buff *skb, } EXPORT_SYMBOL(build_skb_around); +/** + * __napi_build_skb - build a network buffer + * @data: data buffer provided by caller + * @frag_size: size of data, or 0 if head was kmalloced + * + * Version of __build_skb() that uses NAPI percpu caches to obtain + * skbuff_head instead of inplace allocation. + * + * Returns a new &sk_buff on success, %NULL on allocation failure. + */ +static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size) +{ + struct sk_buff *skb; + + skb = napi_skb_cache_get(); + if (unlikely(!skb)) + return NULL; + + memset(skb, 0, offsetof(struct sk_buff, tail)); + __build_skb_around(skb, data, frag_size); + + return skb; +} + +/** + * napi_build_skb - build a network buffer + * @data: data buffer provided by caller + * @frag_size: size of data, or 0 if head was kmalloced + * + * Version of __napi_build_skb() that takes care of skb->head_frag + * and skb->pfmemalloc when the data is a page or page fragment. + * + * Returns a new &sk_buff on success, %NULL on allocation failure. + */ +struct sk_buff *napi_build_skb(void *data, unsigned int frag_size) +{ + struct sk_buff *skb = __napi_build_skb(data, frag_size); + + if (likely(skb) && frag_size) { + skb->head_frag = 1; + skb_propagate_pfmemalloc(virt_to_head_page(data), skb); + } + + return skb; +} +EXPORT_SYMBOL(napi_build_skb); + /* * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells * the caller if emergency pfmemalloc reserves are being used. If it is and @@ -838,31 +906,31 @@ void __consume_stateless_skb(struct sk_buff *skb) kfree_skbmem(skb); } -static inline void _kfree_skb_defer(struct sk_buff *skb) +static void napi_skb_cache_put(struct sk_buff *skb) { struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); + u32 i; /* drop skb->head and call any destructors for packet */ skb_release_all(skb); - /* record skb to CPU local list */ + kasan_poison_object_data(skbuff_head_cache, skb); nc->skb_cache[nc->skb_count++] = skb; -#ifdef CONFIG_SLUB - /* SLUB writes into objects when freeing */ - prefetchw(skb); -#endif - - /* flush skb_cache if it is filled */ if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { - kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE, - nc->skb_cache); - nc->skb_count = 0; + for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++) + kasan_unpoison_object_data(skbuff_head_cache, + nc->skb_cache[i]); + + kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_HALF, + nc->skb_cache + NAPI_SKB_CACHE_HALF); + nc->skb_count = NAPI_SKB_CACHE_HALF; } } + void __kfree_skb_defer(struct sk_buff *skb) { - _kfree_skb_defer(skb); + napi_skb_cache_put(skb); } void napi_consume_skb(struct sk_buff *skb, int budget) @@ -887,7 +955,7 @@ void napi_consume_skb(struct sk_buff *skb, int budget) return; } - _kfree_skb_defer(skb); + napi_skb_cache_put(skb); } EXPORT_SYMBOL(napi_consume_skb); -- cgit v1.2.3 From 9243adfc311a20371c3f4d8eaf0af4b135e6fac3 Mon Sep 17 00:00:00 2001 From: Alexander Lobakin Date: Sat, 13 Feb 2021 14:13:09 +0000 Subject: skbuff: queue NAPI_MERGED_FREE skbs into NAPI cache instead of freeing napi_frags_finish() and napi_skb_finish() can only be called inside NAPI Rx context, so we can feed NAPI cache with skbuff_heads that got NAPI_MERGED_FREE verdict instead of immediate freeing. Replace __kfree_skb() with __kfree_skb_defer() in napi_skb_finish() and move napi_skb_free_stolen_head() to skbuff.c, so it can drop skbs to NAPI cache. As many drivers call napi_alloc_skb()/napi_get_frags() on their receive path, this becomes especially useful. Signed-off-by: Alexander Lobakin Signed-off-by: David S. Miller --- include/linux/skbuff.h | 1 + net/core/dev.c | 9 +-------- net/core/skbuff.c | 12 +++++++++--- 3 files changed, 11 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 906122eac82a..6d0a33d1c0db 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2921,6 +2921,7 @@ static inline struct sk_buff *napi_alloc_skb(struct napi_struct *napi, } void napi_consume_skb(struct sk_buff *skb, int budget); +void napi_skb_free_stolen_head(struct sk_buff *skb); void __kfree_skb_defer(struct sk_buff *skb); /** diff --git a/net/core/dev.c b/net/core/dev.c index 631807c196ad..ea9b46318d23 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -6095,13 +6095,6 @@ struct packet_offload *gro_find_complete_by_type(__be16 type) } EXPORT_SYMBOL(gro_find_complete_by_type); -static void napi_skb_free_stolen_head(struct sk_buff *skb) -{ - skb_dst_drop(skb); - skb_ext_put(skb); - kmem_cache_free(skbuff_head_cache, skb); -} - static gro_result_t napi_skb_finish(struct napi_struct *napi, struct sk_buff *skb, gro_result_t ret) @@ -6115,7 +6108,7 @@ static gro_result_t napi_skb_finish(struct napi_struct *napi, if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) napi_skb_free_stolen_head(skb); else - __kfree_skb(skb); + __kfree_skb_defer(skb); break; case GRO_HELD: diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 875e1a453f7e..545a472273a5 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -916,9 +916,6 @@ static void napi_skb_cache_put(struct sk_buff *skb) struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); u32 i; - /* drop skb->head and call any destructors for packet */ - skb_release_all(skb); - kasan_poison_object_data(skbuff_head_cache, skb); nc->skb_cache[nc->skb_count++] = skb; @@ -935,6 +932,14 @@ static void napi_skb_cache_put(struct sk_buff *skb) void __kfree_skb_defer(struct sk_buff *skb) { + skb_release_all(skb); + napi_skb_cache_put(skb); +} + +void napi_skb_free_stolen_head(struct sk_buff *skb) +{ + skb_dst_drop(skb); + skb_ext_put(skb); napi_skb_cache_put(skb); } @@ -960,6 +965,7 @@ void napi_consume_skb(struct sk_buff *skb, int budget) return; } + skb_release_all(skb); napi_skb_cache_put(skb); } EXPORT_SYMBOL(napi_consume_skb); -- cgit v1.2.3 From 40d3f295b5feda409784e569550057b5fbc2a295 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Sun, 14 Feb 2021 00:37:56 +0200 Subject: net: mscc: ocelot: use common tag parsing code with DSA The Injection Frame Header and Extraction Frame Header that the switch prepends to frames over the NPI port is also prepended to frames delivered over the CPU port module's queues. Let's unify the handling of the frame headers by making the ocelot driver call some helpers exported by the DSA tagger. Among other things, this allows us to get rid of the strange cpu_to_be32 when transmitting the Injection Frame Header on ocelot, since the packing API uses network byte order natively (when "quirks" is 0). The comments above ocelot_gen_ifh talk about setting pop_cnt to 3, and the cpu extraction queue mask to something, but the code doesn't do it, so we don't do it either. Signed-off-by: Vladimir Oltean Reviewed-by: Florian Fainelli Signed-off-by: David S. Miller --- drivers/net/dsa/ocelot/felix.c | 6 +- drivers/net/dsa/ocelot/felix_vsc9959.c | 2 +- drivers/net/dsa/ocelot/seville_vsc9953.c | 2 +- drivers/net/ethernet/mscc/ocelot.c | 38 ++---- drivers/net/ethernet/mscc/ocelot.h | 9 -- drivers/net/ethernet/mscc/ocelot_vsc7514.c | 54 +++----- include/linux/dsa/ocelot.h | 208 +++++++++++++++++++++++++++++ include/soc/mscc/ocelot.h | 7 - net/dsa/tag_ocelot.c | 147 ++------------------ 9 files changed, 246 insertions(+), 227 deletions(-) create mode 100644 include/linux/dsa/ocelot.h (limited to 'include/linux') diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c index d3180b0f2307..60e0d354adea 100644 --- a/drivers/net/dsa/ocelot/felix.c +++ b/drivers/net/dsa/ocelot/felix.c @@ -14,8 +14,8 @@ #include #include #include +#include #include -#include #include #include #include @@ -1161,9 +1161,9 @@ static int felix_hwtstamp_set(struct dsa_switch *ds, int port, static bool felix_rxtstamp(struct dsa_switch *ds, int port, struct sk_buff *skb, unsigned int type) { + u8 *extraction = skb->data - ETH_HLEN - OCELOT_TAG_LEN; struct skb_shared_hwtstamps *shhwtstamps; struct ocelot *ocelot = ds->priv; - u8 *extraction = skb->data - ETH_HLEN - OCELOT_TAG_LEN; u32 tstamp_lo, tstamp_hi; struct timespec64 ts; u64 tstamp, val; @@ -1171,7 +1171,7 @@ static bool felix_rxtstamp(struct dsa_switch *ds, int port, ocelot_ptp_gettime64(&ocelot->ptp_info, &ts); tstamp = ktime_set(ts.tv_sec, ts.tv_nsec); - packing(extraction, &val, 116, 85, OCELOT_TAG_LEN, UNPACK, 0); + ocelot_xfh_get_rew_val(extraction, &val); tstamp_lo = (u32)val; tstamp_hi = tstamp >> 32; diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index e944868cc120..cacc6f9c0113 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c index 512f677a6c1c..d7348ea4831e 100644 --- a/drivers/net/dsa/ocelot/seville_vsc9953.c +++ b/drivers/net/dsa/ocelot/seville_vsc9953.c @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include "felix.h" diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index 7106d9ee534a..699b0c1c1780 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -4,6 +4,7 @@ * * Copyright (c) 2017 Microsemi Corporation */ +#include #include #include #include "ocelot.h" @@ -628,26 +629,6 @@ void ocelot_get_txtstamp(struct ocelot *ocelot) } EXPORT_SYMBOL(ocelot_get_txtstamp); -/* Generate the IFH for frame injection - * - * The IFH is a 128bit-value - * bit 127: bypass the analyzer processing - * bit 56-67: destination mask - * bit 28-29: pop_cnt: 3 disables all rewriting of the frame - * bit 20-27: cpu extraction queue mask - * bit 16: tag type 0: C-tag, 1: S-tag - * bit 0-11: VID - */ -static int ocelot_gen_ifh(u32 *ifh, struct frame_info *info) -{ - ifh[0] = IFH_INJ_BYPASS | ((0x1ff & info->rew_op) << 21); - ifh[1] = (0xf00 & info->port) >> 8; - ifh[2] = (0xff & info->port) << 24; - ifh[3] = (info->tag_type << 16) | info->vid; - - return 0; -} - bool ocelot_can_inject(struct ocelot *ocelot, int grp) { u32 val = ocelot_read(ocelot, QS_INJ_STATUS); @@ -664,23 +645,20 @@ EXPORT_SYMBOL(ocelot_can_inject); void ocelot_port_inject_frame(struct ocelot *ocelot, int port, int grp, u32 rew_op, struct sk_buff *skb) { - struct frame_info info = {}; - u32 ifh[OCELOT_TAG_LEN / 4]; + u32 ifh[OCELOT_TAG_LEN / 4] = {0}; unsigned int i, count, last; ocelot_write_rix(ocelot, QS_INJ_CTRL_GAP_SIZE(1) | QS_INJ_CTRL_SOF, QS_INJ_CTRL, grp); - info.port = BIT(port); - info.tag_type = IFH_TAG_TYPE_C; - info.vid = skb_vlan_tag_get(skb); - info.rew_op = rew_op; - - ocelot_gen_ifh(ifh, &info); + ocelot_ifh_set_bypass(ifh, 1); + ocelot_ifh_set_dest(ifh, BIT(port)); + ocelot_ifh_set_tag_type(ifh, IFH_TAG_TYPE_C); + ocelot_ifh_set_vid(ifh, skb_vlan_tag_get(skb)); + ocelot_ifh_set_rew_op(ifh, rew_op); for (i = 0; i < OCELOT_TAG_LEN / 4; i++) - ocelot_write_rix(ocelot, (__force u32)cpu_to_be32(ifh[i]), - QS_INJ_WR, grp); + ocelot_write_rix(ocelot, ifh[i], QS_INJ_WR, grp); count = DIV_ROUND_UP(skb->len, 4); last = skb->len % 4; diff --git a/drivers/net/ethernet/mscc/ocelot.h b/drivers/net/ethernet/mscc/ocelot.h index c485795c606b..db6b1a4c3926 100644 --- a/drivers/net/ethernet/mscc/ocelot.h +++ b/drivers/net/ethernet/mscc/ocelot.h @@ -32,15 +32,6 @@ #define OCELOT_PTP_QUEUE_SZ 128 -struct frame_info { - u32 len; - u16 port; - u16 vid; - u8 tag_type; - u16 rew_op; - u32 timestamp; /* rew_val */ -}; - struct ocelot_port_tc { bool block_shared; unsigned long offload_cnt; diff --git a/drivers/net/ethernet/mscc/ocelot_vsc7514.c b/drivers/net/ethernet/mscc/ocelot_vsc7514.c index b075dc13354a..fe0f8d6a32ce 100644 --- a/drivers/net/ethernet/mscc/ocelot_vsc7514.c +++ b/drivers/net/ethernet/mscc/ocelot_vsc7514.c @@ -4,6 +4,7 @@ * * Copyright (c) 2017 Microsemi Corporation */ +#include #include #include #include @@ -18,8 +19,6 @@ #include #include "ocelot.h" -#define IFH_EXTRACT_BITFIELD64(x, o, w) (((x) >> (o)) & GENMASK_ULL((w) - 1, 0)) - static const u32 ocelot_ana_regmap[] = { REG(ANA_ADVLEARN, 0x009000), REG(ANA_VLANMASK, 0x009004), @@ -532,29 +531,6 @@ static int ocelot_chip_init(struct ocelot *ocelot, const struct ocelot_ops *ops) return 0; } -static int ocelot_parse_ifh(u32 *_ifh, struct frame_info *info) -{ - u8 llen, wlen; - u64 ifh[2]; - - ifh[0] = be64_to_cpu(((__force __be64 *)_ifh)[0]); - ifh[1] = be64_to_cpu(((__force __be64 *)_ifh)[1]); - - wlen = IFH_EXTRACT_BITFIELD64(ifh[0], 7, 8); - llen = IFH_EXTRACT_BITFIELD64(ifh[0], 15, 6); - - info->len = OCELOT_BUFFER_CELL_SZ * wlen + llen - 80; - - info->timestamp = IFH_EXTRACT_BITFIELD64(ifh[0], 21, 32); - - info->port = IFH_EXTRACT_BITFIELD64(ifh[1], 43, 4); - - info->tag_type = IFH_EXTRACT_BITFIELD64(ifh[1], 16, 1); - info->vid = IFH_EXTRACT_BITFIELD64(ifh[1], 0, 12); - - return 0; -} - static int ocelot_rx_frame_word(struct ocelot *ocelot, u8 grp, bool ifh, u32 *rval) { @@ -609,20 +585,20 @@ static irqreturn_t ocelot_xtr_irq_handler(int irq, void *arg) struct ocelot_port_private *priv; struct ocelot_port *ocelot_port; u64 tod_in_ns, full_ts_in_ns; - struct frame_info info = {}; + u64 src_port, len, timestamp; struct net_device *dev; - u32 ifh[4], val, *buf; + u32 xfh[4], val, *buf; struct timespec64 ts; - int sz, len, buf_len; struct sk_buff *skb; + int sz, buf_len; for (i = 0; i < OCELOT_TAG_LEN / 4; i++) { - err = ocelot_rx_frame_word(ocelot, grp, true, &ifh[i]); + err = ocelot_rx_frame_word(ocelot, grp, true, &xfh[i]); if (err != 4) goto out; } - /* At this point the IFH was read correctly, so it is safe to + /* At this point the XFH was read correctly, so it is safe to * presume that there is no error. The err needs to be reset * otherwise a frame could come in CPU queue between the while * condition and the check for error later on. And in that case @@ -630,21 +606,23 @@ static irqreturn_t ocelot_xtr_irq_handler(int irq, void *arg) */ err = 0; - ocelot_parse_ifh(ifh, &info); + ocelot_xfh_get_src_port(xfh, &src_port); + ocelot_xfh_get_len(xfh, &len); + ocelot_xfh_get_rew_val(xfh, ×tamp); - ocelot_port = ocelot->ports[info.port]; + ocelot_port = ocelot->ports[src_port]; priv = container_of(ocelot_port, struct ocelot_port_private, port); dev = priv->dev; - skb = netdev_alloc_skb(dev, info.len); + skb = netdev_alloc_skb(dev, len); if (unlikely(!skb)) { netdev_err(dev, "Unable to allocate sk_buff\n"); err = -ENOMEM; goto out; } - buf_len = info.len - ETH_FCS_LEN; + buf_len = len - ETH_FCS_LEN; buf = (u32 *)skb_put(skb, buf_len); len = 0; @@ -677,12 +655,12 @@ static irqreturn_t ocelot_xtr_irq_handler(int irq, void *arg) ocelot_ptp_gettime64(&ocelot->ptp_info, &ts); tod_in_ns = ktime_set(ts.tv_sec, ts.tv_nsec); - if ((tod_in_ns & 0xffffffff) < info.timestamp) + if ((tod_in_ns & 0xffffffff) < timestamp) full_ts_in_ns = (((tod_in_ns >> 32) - 1) << 32) | - info.timestamp; + timestamp; else full_ts_in_ns = (tod_in_ns & GENMASK_ULL(63, 32)) | - info.timestamp; + timestamp; shhwtstamps = skb_hwtstamps(skb); memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps)); @@ -692,7 +670,7 @@ static irqreturn_t ocelot_xtr_irq_handler(int irq, void *arg) /* Everything we see on an interface that is in the HW bridge * has already been forwarded. */ - if (ocelot->bridge_mask & BIT(info.port)) + if (ocelot->bridge_mask & BIT(src_port)) skb->offload_fwd_mark = 1; skb->protocol = eth_type_trans(skb, dev); diff --git a/include/linux/dsa/ocelot.h b/include/linux/dsa/ocelot.h new file mode 100644 index 000000000000..763dbbfaff7a --- /dev/null +++ b/include/linux/dsa/ocelot.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: GPL-2.0 + * Copyright 2019-2021 NXP Semiconductors + */ + +#ifndef _NET_DSA_TAG_OCELOT_H +#define _NET_DSA_TAG_OCELOT_H + +#include + +#define OCELOT_TAG_LEN 16 +#define OCELOT_SHORT_PREFIX_LEN 4 +#define OCELOT_LONG_PREFIX_LEN 16 +#define OCELOT_TOTAL_TAG_LEN (OCELOT_SHORT_PREFIX_LEN + OCELOT_TAG_LEN) + +/* The CPU injection header and the CPU extraction header can have 3 types of + * prefixes: long, short and no prefix. The format of the header itself is the + * same in all 3 cases. + * + * Extraction with long prefix: + * + * +-------------------+-------------------+------+------+------------+-------+ + * | ff:ff:ff:ff:ff:ff | fe:ff:ff:ff:ff:ff | 8880 | 000a | extraction | frame | + * | | | | | header | | + * +-------------------+-------------------+------+------+------------+-------+ + * 48 bits 48 bits 16 bits 16 bits 128 bits + * + * Extraction with short prefix: + * + * +------+------+------------+-------+ + * | 8880 | 000a | extraction | frame | + * | | | header | | + * +------+------+------------+-------+ + * 16 bits 16 bits 128 bits + * + * Extraction with no prefix: + * + * +------------+-------+ + * | extraction | frame | + * | header | | + * +------------+-------+ + * 128 bits + * + * + * Injection with long prefix: + * + * +-------------------+-------------------+------+------+------------+-------+ + * | any dmac | any smac | 8880 | 000a | injection | frame | + * | | | | | header | | + * +-------------------+-------------------+------+------+------------+-------+ + * 48 bits 48 bits 16 bits 16 bits 128 bits + * + * Injection with short prefix: + * + * +------+------+------------+-------+ + * | 8880 | 000a | injection | frame | + * | | | header | | + * +------+------+------------+-------+ + * 16 bits 16 bits 128 bits + * + * Injection with no prefix: + * + * +------------+-------+ + * | injection | frame | + * | header | | + * +------------+-------+ + * 128 bits + * + * The injection header looks like this (network byte order, bit 127 + * is part of lowest address byte in memory, bit 0 is part of highest + * address byte): + * + * +------+------+------+------+------+------+------+------+ + * 127:120 |BYPASS| MASQ | MASQ_PORT |REW_OP|REW_OP| + * +------+------+------+------+------+------+------+------+ + * 119:112 | REW_OP | + * +------+------+------+------+------+------+------+------+ + * 111:104 | REW_VAL | + * +------+------+------+------+------+------+------+------+ + * 103: 96 | REW_VAL | + * +------+------+------+------+------+------+------+------+ + * 95: 88 | REW_VAL | + * +------+------+------+------+------+------+------+------+ + * 87: 80 | REW_VAL | + * +------+------+------+------+------+------+------+------+ + * 79: 72 | RSV | + * +------+------+------+------+------+------+------+------+ + * 71: 64 | RSV | DEST | + * +------+------+------+------+------+------+------+------+ + * 63: 56 | DEST | + * +------+------+------+------+------+------+------+------+ + * 55: 48 | RSV | + * +------+------+------+------+------+------+------+------+ + * 47: 40 | RSV | SRC_PORT | RSV |TFRM_TIMER| + * +------+------+------+------+------+------+------+------+ + * 39: 32 | TFRM_TIMER | RSV | + * +------+------+------+------+------+------+------+------+ + * 31: 24 | RSV | DP | POP_CNT | CPUQ | + * +------+------+------+------+------+------+------+------+ + * 23: 16 | CPUQ | QOS_CLASS |TAG_TYPE| + * +------+------+------+------+------+------+------+------+ + * 15: 8 | PCP | DEI | VID | + * +------+------+------+------+------+------+------+------+ + * 7: 0 | VID | + * +------+------+------+------+------+------+------+------+ + * + * And the extraction header looks like this: + * + * +------+------+------+------+------+------+------+------+ + * 127:120 | RSV | REW_OP | + * +------+------+------+------+------+------+------+------+ + * 119:112 | REW_OP | REW_VAL | + * +------+------+------+------+------+------+------+------+ + * 111:104 | REW_VAL | + * +------+------+------+------+------+------+------+------+ + * 103: 96 | REW_VAL | + * +------+------+------+------+------+------+------+------+ + * 95: 88 | REW_VAL | + * +------+------+------+------+------+------+------+------+ + * 87: 80 | REW_VAL | LLEN | + * +------+------+------+------+------+------+------+------+ + * 79: 72 | LLEN | WLEN | + * +------+------+------+------+------+------+------+------+ + * 71: 64 | WLEN | RSV | + * +------+------+------+------+------+------+------+------+ + * 63: 56 | RSV | + * +------+------+------+------+------+------+------+------+ + * 55: 48 | RSV | + * +------+------+------+------+------+------+------+------+ + * 47: 40 | RSV | SRC_PORT | ACL_ID | + * +------+------+------+------+------+------+------+------+ + * 39: 32 | ACL_ID | RSV | SFLOW_ID | + * +------+------+------+------+------+------+------+------+ + * 31: 24 |ACL_HIT| DP | LRN_FLAGS | CPUQ | + * +------+------+------+------+------+------+------+------+ + * 23: 16 | CPUQ | QOS_CLASS |TAG_TYPE| + * +------+------+------+------+------+------+------+------+ + * 15: 8 | PCP | DEI | VID | + * +------+------+------+------+------+------+------+------+ + * 7: 0 | VID | + * +------+------+------+------+------+------+------+------+ + */ + +static inline void ocelot_xfh_get_rew_val(void *extraction, u64 *rew_val) +{ + packing(extraction, rew_val, 116, 85, OCELOT_TAG_LEN, UNPACK, 0); +} + +static inline void ocelot_xfh_get_len(void *extraction, u64 *len) +{ + u64 llen, wlen; + + packing(extraction, &llen, 84, 79, OCELOT_TAG_LEN, UNPACK, 0); + packing(extraction, &wlen, 78, 71, OCELOT_TAG_LEN, UNPACK, 0); + + *len = 60 * wlen + llen - 80; +} + +static inline void ocelot_xfh_get_src_port(void *extraction, u64 *src_port) +{ + packing(extraction, src_port, 46, 43, OCELOT_TAG_LEN, UNPACK, 0); +} + +static inline void ocelot_xfh_get_qos_class(void *extraction, u64 *qos_class) +{ + packing(extraction, qos_class, 19, 17, OCELOT_TAG_LEN, UNPACK, 0); +} + +static inline void ocelot_xfh_get_tag_type(void *extraction, u64 *tag_type) +{ + packing(extraction, tag_type, 16, 16, OCELOT_TAG_LEN, UNPACK, 0); +} + +static inline void ocelot_xfh_get_vlan_tci(void *extraction, u64 *vlan_tci) +{ + packing(extraction, vlan_tci, 15, 0, OCELOT_TAG_LEN, UNPACK, 0); +} + +static inline void ocelot_ifh_set_bypass(void *injection, u64 bypass) +{ + packing(injection, &bypass, 127, 127, OCELOT_TAG_LEN, PACK, 0); +} + +static inline void ocelot_ifh_set_rew_op(void *injection, u64 rew_op) +{ + packing(injection, &rew_op, 125, 117, OCELOT_TAG_LEN, PACK, 0); +} + +static inline void ocelot_ifh_set_dest(void *injection, u64 dest) +{ + packing(injection, &dest, 67, 56, OCELOT_TAG_LEN, PACK, 0); +} + +static inline void ocelot_ifh_set_qos_class(void *injection, u64 qos_class) +{ + packing(injection, &qos_class, 19, 17, OCELOT_TAG_LEN, PACK, 0); +} + +static inline void ocelot_ifh_set_tag_type(void *injection, u64 tag_type) +{ + packing(injection, &tag_type, 16, 16, OCELOT_TAG_LEN, PACK, 0); +} + +static inline void ocelot_ifh_set_vid(void *injection, u64 vid) +{ + packing(injection, &vid, 11, 0, OCELOT_TAG_LEN, PACK, 0); +} + +#endif diff --git a/include/soc/mscc/ocelot.h b/include/soc/mscc/ocelot.h index a22aa944e921..11cae2e968b5 100644 --- a/include/soc/mscc/ocelot.h +++ b/include/soc/mscc/ocelot.h @@ -87,9 +87,6 @@ /* Source PGIDs, one per physical port */ #define PGID_SRC 80 -#define IFH_INJ_BYPASS BIT(31) -#define IFH_INJ_POP_CNT_DISABLE (3 << 28) - #define IFH_TAG_TYPE_C 0 #define IFH_TAG_TYPE_S 1 @@ -100,10 +97,6 @@ #define IFH_REW_OP_ORIGIN_PTP 0x5 #define OCELOT_NUM_TC 8 -#define OCELOT_TAG_LEN 16 -#define OCELOT_SHORT_PREFIX_LEN 4 -#define OCELOT_LONG_PREFIX_LEN 16 -#define OCELOT_TOTAL_TAG_LEN (OCELOT_SHORT_PREFIX_LEN + OCELOT_TAG_LEN) #define OCELOT_SPEED_2500 0 #define OCELOT_SPEED_1000 1 diff --git a/net/dsa/tag_ocelot.c b/net/dsa/tag_ocelot.c index 225b145fd131..8ce0b26f3520 100644 --- a/net/dsa/tag_ocelot.c +++ b/net/dsa/tag_ocelot.c @@ -1,138 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright 2019 NXP Semiconductors */ +#include #include -#include #include "dsa_priv.h" -/* The CPU injection header and the CPU extraction header can have 3 types of - * prefixes: long, short and no prefix. The format of the header itself is the - * same in all 3 cases. - * - * Extraction with long prefix: - * - * +-------------------+-------------------+------+------+------------+-------+ - * | ff:ff:ff:ff:ff:ff | ff:ff:ff:ff:ff:ff | 8880 | 000a | extraction | frame | - * | | | | | header | | - * +-------------------+-------------------+------+------+------------+-------+ - * 48 bits 48 bits 16 bits 16 bits 128 bits - * - * Extraction with short prefix: - * - * +------+------+------------+-------+ - * | 8880 | 000a | extraction | frame | - * | | | header | | - * +------+------+------------+-------+ - * 16 bits 16 bits 128 bits - * - * Extraction with no prefix: - * - * +------------+-------+ - * | extraction | frame | - * | header | | - * +------------+-------+ - * 128 bits - * - * - * Injection with long prefix: - * - * +-------------------+-------------------+------+------+------------+-------+ - * | any dmac | any smac | 8880 | 000a | injection | frame | - * | | | | | header | | - * +-------------------+-------------------+------+------+------------+-------+ - * 48 bits 48 bits 16 bits 16 bits 128 bits - * - * Injection with short prefix: - * - * +------+------+------------+-------+ - * | 8880 | 000a | injection | frame | - * | | | header | | - * +------+------+------------+-------+ - * 16 bits 16 bits 128 bits - * - * Injection with no prefix: - * - * +------------+-------+ - * | injection | frame | - * | header | | - * +------------+-------+ - * 128 bits - * - * The injection header looks like this (network byte order, bit 127 - * is part of lowest address byte in memory, bit 0 is part of highest - * address byte): - * - * +------+------+------+------+------+------+------+------+ - * 127:120 |BYPASS| MASQ | MASQ_PORT |REW_OP|REW_OP| - * +------+------+------+------+------+------+------+------+ - * 119:112 | REW_OP | - * +------+------+------+------+------+------+------+------+ - * 111:104 | REW_VAL | - * +------+------+------+------+------+------+------+------+ - * 103: 96 | REW_VAL | - * +------+------+------+------+------+------+------+------+ - * 95: 88 | REW_VAL | - * +------+------+------+------+------+------+------+------+ - * 87: 80 | REW_VAL | - * +------+------+------+------+------+------+------+------+ - * 79: 72 | RSV | - * +------+------+------+------+------+------+------+------+ - * 71: 64 | RSV | DEST | - * +------+------+------+------+------+------+------+------+ - * 63: 56 | DEST | - * +------+------+------+------+------+------+------+------+ - * 55: 48 | RSV | - * +------+------+------+------+------+------+------+------+ - * 47: 40 | RSV | SRC_PORT | RSV |TFRM_TIMER| - * +------+------+------+------+------+------+------+------+ - * 39: 32 | TFRM_TIMER | RSV | - * +------+------+------+------+------+------+------+------+ - * 31: 24 | RSV | DP | POP_CNT | CPUQ | - * +------+------+------+------+------+------+------+------+ - * 23: 16 | CPUQ | QOS_CLASS |TAG_TYPE| - * +------+------+------+------+------+------+------+------+ - * 15: 8 | PCP | DEI | VID | - * +------+------+------+------+------+------+------+------+ - * 7: 0 | VID | - * +------+------+------+------+------+------+------+------+ - * - * And the extraction header looks like this: - * - * +------+------+------+------+------+------+------+------+ - * 127:120 | RSV | REW_OP | - * +------+------+------+------+------+------+------+------+ - * 119:112 | REW_OP | REW_VAL | - * +------+------+------+------+------+------+------+------+ - * 111:104 | REW_VAL | - * +------+------+------+------+------+------+------+------+ - * 103: 96 | REW_VAL | - * +------+------+------+------+------+------+------+------+ - * 95: 88 | REW_VAL | - * +------+------+------+------+------+------+------+------+ - * 87: 80 | REW_VAL | LLEN | - * +------+------+------+------+------+------+------+------+ - * 79: 72 | LLEN | WLEN | - * +------+------+------+------+------+------+------+------+ - * 71: 64 | WLEN | RSV | - * +------+------+------+------+------+------+------+------+ - * 63: 56 | RSV | - * +------+------+------+------+------+------+------+------+ - * 55: 48 | RSV | - * +------+------+------+------+------+------+------+------+ - * 47: 40 | RSV | SRC_PORT | ACL_ID | - * +------+------+------+------+------+------+------+------+ - * 39: 32 | ACL_ID | RSV | SFLOW_ID | - * +------+------+------+------+------+------+------+------+ - * 31: 24 |ACL_HIT| DP | LRN_FLAGS | CPUQ | - * +------+------+------+------+------+------+------+------+ - * 23: 16 | CPUQ | QOS_CLASS |TAG_TYPE| - * +------+------+------+------+------+------+------+------+ - * 15: 8 | PCP | DEI | VID | - * +------+------+------+------+------+------+------+------+ - * 7: 0 | VID | - * +------+------+------+------+------+------+------+------+ - */ - static struct sk_buff *ocelot_xmit(struct sk_buff *skb, struct net_device *netdev) { @@ -142,7 +14,6 @@ static struct sk_buff *ocelot_xmit(struct sk_buff *skb, struct ocelot *ocelot = ds->priv; struct ocelot_port *ocelot_port; u8 *prefix, *injection; - u64 qos_class, rew_op; ocelot_port = ocelot->ports[dp->index]; @@ -155,19 +26,19 @@ static struct sk_buff *ocelot_xmit(struct sk_buff *skb, /* Fix up the fields which are not statically determined * in the template */ - qos_class = skb->priority; - packing(injection, &qos_class, 19, 17, OCELOT_TAG_LEN, PACK, 0); + ocelot_ifh_set_qos_class(injection, skb->priority); /* TX timestamping was requested */ if (clone) { - rew_op = ocelot_port->ptp_cmd; + u64 rew_op = ocelot_port->ptp_cmd; + /* Retrieve timestamp ID populated inside skb->cb[0] of the * clone by ocelot_port_add_txtstamp_skb */ if (ocelot_port->ptp_cmd == IFH_REW_OP_TWO_STEP_PTP) rew_op |= clone->cb[0] << 3; - packing(injection, &rew_op, 125, 117, OCELOT_TAG_LEN, PACK, 0); + ocelot_ifh_set_rew_op(injection, rew_op); } return skb; @@ -208,10 +79,10 @@ static struct sk_buff *ocelot_rcv(struct sk_buff *skb, /* Remove from inet csum the extraction header */ skb_postpull_rcsum(skb, start, OCELOT_TOTAL_TAG_LEN); - packing(extraction, &src_port, 46, 43, OCELOT_TAG_LEN, UNPACK, 0); - packing(extraction, &qos_class, 19, 17, OCELOT_TAG_LEN, UNPACK, 0); - packing(extraction, &tag_type, 16, 16, OCELOT_TAG_LEN, UNPACK, 0); - packing(extraction, &vlan_tci, 15, 0, OCELOT_TAG_LEN, UNPACK, 0); + ocelot_xfh_get_src_port(extraction, &src_port); + ocelot_xfh_get_qos_class(extraction, &qos_class); + ocelot_xfh_get_tag_type(extraction, &tag_type); + ocelot_xfh_get_vlan_tci(extraction, &vlan_tci); skb->dev = dsa_master_find_slave(netdev, 0, src_port); if (!skb->dev) -- cgit v1.2.3 From 7c4bb540e9173c914c2091fdd9b6aee3c2a3e1e5 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Sun, 14 Feb 2021 00:37:58 +0200 Subject: net: dsa: tag_ocelot: create separate tagger for Seville The ocelot tagger is a hot mess currently, it relies on memory initialized by the attached driver for basic frame transmission. This is against all that DSA tagging protocols stand for, which is that the transmission and reception of a DSA-tagged frame, the data path, should be independent from the switch control path, because the tag protocol is in principle hot-pluggable and reusable across switches (even if in practice it wasn't until very recently). But if another driver like dsa_loop wants to make use of tag_ocelot, it couldn't. This was done to have common code between Felix and Ocelot, which have one bit difference in the frame header format. Quoting from commit 67c2404922c2 ("net: dsa: felix: create a template for the DSA tags on xmit"): Other alternatives have been analyzed, such as: - Create a separate tag_seville.c: too much code duplication for just 1 bit field difference. - Create a separate DSA_TAG_PROTO_SEVILLE under tag_ocelot.c, just like tag_brcm.c, which would have a separate .xmit function. Again, too much code duplication for just 1 bit field difference. - Allocate the template from the init function of the tag_ocelot.c module, instead of from the driver: couldn't figure out a method of accessing the correct port template corresponding to the correct tagger in the .xmit function. The really interesting part is that Seville should have had its own tagging protocol defined - it is not compatible on the wire with Ocelot, even for that single bit. In principle, a packet generated by DSA_TAG_PROTO_OCELOT when booted on NXP LS1028A would look in a certain way, but when booted on NXP T1040 it would look differently. The reverse is also true: a packet generated by a Seville switch would be interpreted incorrectly by Wireshark if it was told it was generated by an Ocelot switch. Actually things are a bit more nuanced. If we concentrate only on the DSA tag, what I said above is true, but Ocelot/Seville also support an optional DSA tag prefix, which can be short or long, and it is possible to distinguish the two taggers based on an integer constant put in that prefix. Nonetheless, creating a separate tagger is still justified, since the tag prefix is optional, and without it, there is again no way to distinguish. Claiming backwards binary compatibility is a bit more tough, since I've already changed the format of tag_ocelot once, in commit 5124197ce58b ("net: dsa: tag_ocelot: use a short prefix on both ingress and egress"). Therefore I am not very concerned with treating this as a bugfix and backporting it to stable kernels (which would be another mess due to the fact that there would be lots of conflicts with the other DSA_TAG_PROTO* definitions). It's just simpler to say that the string values of the taggers have ABI value starting with kernel 5.12, which will be when the changing of tag protocol via /sys/class/net//dsa/tagging goes live. Signed-off-by: Vladimir Oltean Reviewed-by: Florian Fainelli Signed-off-by: David S. Miller --- drivers/net/dsa/ocelot/felix.c | 18 ++------- drivers/net/dsa/ocelot/felix.h | 1 - drivers/net/dsa/ocelot/felix_vsc9959.c | 26 ------------ drivers/net/dsa/ocelot/seville_vsc9953.c | 28 +------------ include/linux/dsa/ocelot.h | 10 +++++ include/net/dsa.h | 2 + net/dsa/tag_ocelot.c | 68 +++++++++++++++++++++++++------- 7 files changed, 70 insertions(+), 83 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c index 60e0d354adea..7e1f6350139b 100644 --- a/drivers/net/dsa/ocelot/felix.c +++ b/drivers/net/dsa/ocelot/felix.c @@ -431,6 +431,7 @@ static int felix_set_tag_protocol(struct dsa_switch *ds, int cpu, int err; switch (proto) { + case DSA_TAG_PROTO_SEVILLE: case DSA_TAG_PROTO_OCELOT: err = felix_setup_tag_npi(ds, cpu); break; @@ -448,6 +449,7 @@ static void felix_del_tag_protocol(struct dsa_switch *ds, int cpu, enum dsa_tag_protocol proto) { switch (proto) { + case DSA_TAG_PROTO_SEVILLE: case DSA_TAG_PROTO_OCELOT: felix_teardown_tag_npi(ds, cpu); break; @@ -471,7 +473,8 @@ static int felix_change_tag_protocol(struct dsa_switch *ds, int cpu, enum dsa_tag_protocol old_proto = felix->tag_proto; int err; - if (proto != DSA_TAG_PROTO_OCELOT && + if (proto != DSA_TAG_PROTO_SEVILLE && + proto != DSA_TAG_PROTO_OCELOT && proto != DSA_TAG_PROTO_OCELOT_8021Q) return -EPROTONOSUPPORT; @@ -1003,7 +1006,6 @@ static int felix_init_structs(struct felix *felix, int num_phys_ports) for (port = 0; port < num_phys_ports; port++) { struct ocelot_port *ocelot_port; struct regmap *target; - u8 *template; ocelot_port = devm_kzalloc(ocelot->dev, sizeof(struct ocelot_port), @@ -1029,22 +1031,10 @@ static int felix_init_structs(struct felix *felix, int num_phys_ports) return PTR_ERR(target); } - template = devm_kzalloc(ocelot->dev, OCELOT_TOTAL_TAG_LEN, - GFP_KERNEL); - if (!template) { - dev_err(ocelot->dev, - "Failed to allocate memory for DSA tag\n"); - kfree(port_phy_modes); - return -ENOMEM; - } - ocelot_port->phy_mode = port_phy_modes[port]; ocelot_port->ocelot = ocelot; ocelot_port->target = target; - ocelot_port->xmit_template = template; ocelot->ports[port] = ocelot_port; - - felix->info->xmit_template_populate(ocelot, port); } kfree(port_phy_modes); diff --git a/drivers/net/dsa/ocelot/felix.h b/drivers/net/dsa/ocelot/felix.h index 9d4459f2fffb..b2ea425c5803 100644 --- a/drivers/net/dsa/ocelot/felix.h +++ b/drivers/net/dsa/ocelot/felix.h @@ -34,7 +34,6 @@ struct felix_info { enum tc_setup_type type, void *type_data); void (*port_sched_speed_set)(struct ocelot *ocelot, int port, u32 speed); - void (*xmit_template_populate)(struct ocelot *ocelot, int port); }; extern const struct dsa_switch_ops felix_switch_ops; diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index cacc6f9c0113..32b885fcaf90 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -1339,31 +1339,6 @@ static int vsc9959_port_setup_tc(struct dsa_switch *ds, int port, } } -static void vsc9959_xmit_template_populate(struct ocelot *ocelot, int port) -{ - struct ocelot_port *ocelot_port = ocelot->ports[port]; - u8 *template = ocelot_port->xmit_template; - u64 bypass, dest, src; - __be32 *prefix; - u8 *injection; - - /* Set the source port as the CPU port module and not the - * NPI port - */ - src = ocelot->num_phys_ports; - dest = BIT(port); - bypass = true; - - injection = template + OCELOT_SHORT_PREFIX_LEN; - prefix = (__be32 *)template; - - packing(injection, &bypass, 127, 127, OCELOT_TAG_LEN, PACK, 0); - packing(injection, &dest, 68, 56, OCELOT_TAG_LEN, PACK, 0); - packing(injection, &src, 46, 43, OCELOT_TAG_LEN, PACK, 0); - - *prefix = cpu_to_be32(0x8880000a); -} - static const struct felix_info felix_info_vsc9959 = { .target_io_res = vsc9959_target_io_res, .port_io_res = vsc9959_port_io_res, @@ -1386,7 +1361,6 @@ static const struct felix_info felix_info_vsc9959 = { .prevalidate_phy_mode = vsc9959_prevalidate_phy_mode, .port_setup_tc = vsc9959_port_setup_tc, .port_sched_speed_set = vsc9959_sched_speed_set, - .xmit_template_populate = vsc9959_xmit_template_populate, }; static irqreturn_t felix_irq_handler(int irq, void *data) diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c index d7348ea4831e..84f93a874d50 100644 --- a/drivers/net/dsa/ocelot/seville_vsc9953.c +++ b/drivers/net/dsa/ocelot/seville_vsc9953.c @@ -1165,31 +1165,6 @@ static void vsc9953_mdio_bus_free(struct ocelot *ocelot) mdiobus_unregister(felix->imdio); } -static void vsc9953_xmit_template_populate(struct ocelot *ocelot, int port) -{ - struct ocelot_port *ocelot_port = ocelot->ports[port]; - u8 *template = ocelot_port->xmit_template; - u64 bypass, dest, src; - __be32 *prefix; - u8 *injection; - - /* Set the source port as the CPU port module and not the - * NPI port - */ - src = ocelot->num_phys_ports; - dest = BIT(port); - bypass = true; - - injection = template + OCELOT_SHORT_PREFIX_LEN; - prefix = (__be32 *)template; - - packing(injection, &bypass, 127, 127, OCELOT_TAG_LEN, PACK, 0); - packing(injection, &dest, 67, 57, OCELOT_TAG_LEN, PACK, 0); - packing(injection, &src, 46, 43, OCELOT_TAG_LEN, PACK, 0); - - *prefix = cpu_to_be32(0x88800005); -} - static const struct felix_info seville_info_vsc9953 = { .target_io_res = vsc9953_target_io_res, .port_io_res = vsc9953_port_io_res, @@ -1206,7 +1181,6 @@ static const struct felix_info seville_info_vsc9953 = { .mdio_bus_free = vsc9953_mdio_bus_free, .phylink_validate = vsc9953_phylink_validate, .prevalidate_phy_mode = vsc9953_prevalidate_phy_mode, - .xmit_template_populate = vsc9953_xmit_template_populate, }; static int seville_probe(struct platform_device *pdev) @@ -1246,7 +1220,7 @@ static int seville_probe(struct platform_device *pdev) ds->ops = &felix_switch_ops; ds->priv = ocelot; felix->ds = ds; - felix->tag_proto = DSA_TAG_PROTO_OCELOT; + felix->tag_proto = DSA_TAG_PROTO_SEVILLE; err = dsa_register_switch(ds); if (err) { diff --git a/include/linux/dsa/ocelot.h b/include/linux/dsa/ocelot.h index 763dbbfaff7a..c6bc45ae5e03 100644 --- a/include/linux/dsa/ocelot.h +++ b/include/linux/dsa/ocelot.h @@ -195,6 +195,16 @@ static inline void ocelot_ifh_set_qos_class(void *injection, u64 qos_class) packing(injection, &qos_class, 19, 17, OCELOT_TAG_LEN, PACK, 0); } +static inline void seville_ifh_set_dest(void *injection, u64 dest) +{ + packing(injection, &dest, 67, 57, OCELOT_TAG_LEN, PACK, 0); +} + +static inline void ocelot_ifh_set_src(void *injection, u64 src) +{ + packing(injection, &src, 46, 43, OCELOT_TAG_LEN, PACK, 0); +} + static inline void ocelot_ifh_set_tag_type(void *injection, u64 tag_type) { packing(injection, &tag_type, 16, 16, OCELOT_TAG_LEN, PACK, 0); diff --git a/include/net/dsa.h b/include/net/dsa.h index 74457aaffec7..b095ef114fe8 100644 --- a/include/net/dsa.h +++ b/include/net/dsa.h @@ -48,6 +48,7 @@ struct phylink_link_state; #define DSA_TAG_PROTO_HELLCREEK_VALUE 18 #define DSA_TAG_PROTO_XRS700X_VALUE 19 #define DSA_TAG_PROTO_OCELOT_8021Q_VALUE 20 +#define DSA_TAG_PROTO_SEVILLE_VALUE 21 enum dsa_tag_protocol { DSA_TAG_PROTO_NONE = DSA_TAG_PROTO_NONE_VALUE, @@ -71,6 +72,7 @@ enum dsa_tag_protocol { DSA_TAG_PROTO_HELLCREEK = DSA_TAG_PROTO_HELLCREEK_VALUE, DSA_TAG_PROTO_XRS700X = DSA_TAG_PROTO_XRS700X_VALUE, DSA_TAG_PROTO_OCELOT_8021Q = DSA_TAG_PROTO_OCELOT_8021Q_VALUE, + DSA_TAG_PROTO_SEVILLE = DSA_TAG_PROTO_SEVILLE_VALUE, }; struct packet_type; diff --git a/net/dsa/tag_ocelot.c b/net/dsa/tag_ocelot.c index fe00519229d7..a7dd61c8e005 100644 --- a/net/dsa/tag_ocelot.c +++ b/net/dsa/tag_ocelot.c @@ -24,33 +24,52 @@ static void ocelot_xmit_ptp(struct dsa_port *dp, void *injection, ocelot_ifh_set_rew_op(injection, rew_op); } -static struct sk_buff *ocelot_xmit(struct sk_buff *skb, - struct net_device *netdev) +static void ocelot_xmit_common(struct sk_buff *skb, struct net_device *netdev, + __be32 ifh_prefix, void **ifh) { struct dsa_port *dp = dsa_slave_to_port(netdev); struct sk_buff *clone = DSA_SKB_CB(skb)->clone; struct dsa_switch *ds = dp->ds; - struct ocelot *ocelot = ds->priv; - struct ocelot_port *ocelot_port; - u8 *prefix, *injection; - - ocelot_port = ocelot->ports[dp->index]; + void *injection; + __be32 *prefix; injection = skb_push(skb, OCELOT_TAG_LEN); - prefix = skb_push(skb, OCELOT_SHORT_PREFIX_LEN); - memcpy(prefix, ocelot_port->xmit_template, OCELOT_TOTAL_TAG_LEN); - - /* Fix up the fields which are not statically determined - * in the template - */ + *prefix = ifh_prefix; + memset(injection, 0, OCELOT_TAG_LEN); + ocelot_ifh_set_bypass(injection, 1); + ocelot_ifh_set_src(injection, ds->num_ports); ocelot_ifh_set_qos_class(injection, skb->priority); /* TX timestamping was requested */ if (clone) ocelot_xmit_ptp(dp, injection, clone); + *ifh = injection; +} + +static struct sk_buff *ocelot_xmit(struct sk_buff *skb, + struct net_device *netdev) +{ + struct dsa_port *dp = dsa_slave_to_port(netdev); + void *injection; + + ocelot_xmit_common(skb, netdev, cpu_to_be32(0x8880000a), &injection); + ocelot_ifh_set_dest(injection, BIT(dp->index)); + + return skb; +} + +static struct sk_buff *seville_xmit(struct sk_buff *skb, + struct net_device *netdev) +{ + struct dsa_port *dp = dsa_slave_to_port(netdev); + void *injection; + + ocelot_xmit_common(skb, netdev, cpu_to_be32(0x88800005), &injection); + seville_ifh_set_dest(injection, BIT(dp->index)); + return skb; } @@ -147,7 +166,26 @@ static const struct dsa_device_ops ocelot_netdev_ops = { .promisc_on_master = true, }; -MODULE_LICENSE("GPL v2"); +DSA_TAG_DRIVER(ocelot_netdev_ops); MODULE_ALIAS_DSA_TAG_DRIVER(DSA_TAG_PROTO_OCELOT); -module_dsa_tag_driver(ocelot_netdev_ops); +static const struct dsa_device_ops seville_netdev_ops = { + .name = "seville", + .proto = DSA_TAG_PROTO_SEVILLE, + .xmit = seville_xmit, + .rcv = ocelot_rcv, + .overhead = OCELOT_TOTAL_TAG_LEN, + .promisc_on_master = true, +}; + +DSA_TAG_DRIVER(seville_netdev_ops); +MODULE_ALIAS_DSA_TAG_DRIVER(DSA_TAG_PROTO_SEVILLE); + +static struct dsa_tag_driver *ocelot_tag_driver_array[] = { + &DSA_TAG_DRIVER_NAME(ocelot_netdev_ops), + &DSA_TAG_DRIVER_NAME(seville_netdev_ops), +}; + +module_dsa_tag_drivers(ocelot_tag_driver_array); + +MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From 3019270282a175defc02c8331786c73e082cd2a8 Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Mon, 15 Feb 2021 10:44:58 +0000 Subject: Revert "Drivers: hv: vmbus: Copy packets sent by Hyper-V out of the ring buffer" This reverts commit a8c3209998afb5c4941b49e35b513cea9050cb4a. It is reported that the said commit caused regression in netvsc. Reported-by: Andrea Parri (Microsoft) Signed-off-by: Wei Liu --- drivers/hv/channel.c | 9 ++--- drivers/hv/hv_fcopy.c | 1 - drivers/hv/hv_kvp.c | 1 - drivers/hv/hyperv_vmbus.h | 2 +- drivers/hv/ring_buffer.c | 82 ++++++--------------------------------- drivers/net/hyperv/hyperv_net.h | 3 -- drivers/net/hyperv/netvsc.c | 2 - drivers/net/hyperv/rndis_filter.c | 2 - drivers/scsi/storvsc_drv.c | 10 ----- include/linux/hyperv.h | 48 ++++------------------- net/vmw_vsock/hyperv_transport.c | 4 +- 11 files changed, 25 insertions(+), 139 deletions(-) (limited to 'include/linux') diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index 9aa789e5f22b..0bd202de7960 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -597,15 +597,12 @@ static int __vmbus_open(struct vmbus_channel *newchannel, newchannel->onchannel_callback = onchannelcallback; newchannel->channel_callback_context = context; - if (!newchannel->max_pkt_size) - newchannel->max_pkt_size = VMBUS_DEFAULT_MAX_PKT_SIZE; - - err = hv_ringbuffer_init(&newchannel->outbound, page, send_pages, 0); + err = hv_ringbuffer_init(&newchannel->outbound, page, send_pages); if (err) goto error_clean_ring; - err = hv_ringbuffer_init(&newchannel->inbound, &page[send_pages], - recv_pages, newchannel->max_pkt_size); + err = hv_ringbuffer_init(&newchannel->inbound, + &page[send_pages], recv_pages); if (err) goto error_clean_ring; diff --git a/drivers/hv/hv_fcopy.c b/drivers/hv/hv_fcopy.c index 660036da7449..59ce85e00a02 100644 --- a/drivers/hv/hv_fcopy.c +++ b/drivers/hv/hv_fcopy.c @@ -349,7 +349,6 @@ int hv_fcopy_init(struct hv_util_service *srv) { recv_buffer = srv->recv_buffer; fcopy_transaction.recv_channel = srv->channel; - fcopy_transaction.recv_channel->max_pkt_size = HV_HYP_PAGE_SIZE * 2; /* * When this driver loads, the user level daemon that diff --git a/drivers/hv/hv_kvp.c b/drivers/hv/hv_kvp.c index c698592b83e4..b49962d312ce 100644 --- a/drivers/hv/hv_kvp.c +++ b/drivers/hv/hv_kvp.c @@ -757,7 +757,6 @@ hv_kvp_init(struct hv_util_service *srv) { recv_buffer = srv->recv_buffer; kvp_transaction.recv_channel = srv->channel; - kvp_transaction.recv_channel->max_pkt_size = HV_HYP_PAGE_SIZE * 4; /* * When this driver loads, the user level daemon that diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h index 42f3d9d123a1..9416e09ebd58 100644 --- a/drivers/hv/hyperv_vmbus.h +++ b/drivers/hv/hyperv_vmbus.h @@ -174,7 +174,7 @@ extern int hv_synic_cleanup(unsigned int cpu); void hv_ringbuffer_pre_init(struct vmbus_channel *channel); int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info, - struct page *pages, u32 pagecnt, u32 max_pkt_size); + struct page *pages, u32 pagecnt); void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info); diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c index 29e90477363a..35833d4d1a1d 100644 --- a/drivers/hv/ring_buffer.c +++ b/drivers/hv/ring_buffer.c @@ -190,7 +190,7 @@ void hv_ringbuffer_pre_init(struct vmbus_channel *channel) /* Initialize the ring buffer. */ int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info, - struct page *pages, u32 page_cnt, u32 max_pkt_size) + struct page *pages, u32 page_cnt) { int i; struct page **pages_wraparound; @@ -232,14 +232,6 @@ int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info, sizeof(struct hv_ring_buffer); ring_info->priv_read_index = 0; - /* Initialize buffer that holds copies of incoming packets */ - if (max_pkt_size) { - ring_info->pkt_buffer = kzalloc(max_pkt_size, GFP_KERNEL); - if (!ring_info->pkt_buffer) - return -ENOMEM; - ring_info->pkt_buffer_size = max_pkt_size; - } - spin_lock_init(&ring_info->ring_lock); return 0; @@ -252,9 +244,6 @@ void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info) vunmap(ring_info->ring_buffer); ring_info->ring_buffer = NULL; mutex_unlock(&ring_info->ring_buffer_mutex); - - kfree(ring_info->pkt_buffer); - ring_info->pkt_buffer_size = 0; } /* Write to the ring buffer. */ @@ -396,7 +385,7 @@ int hv_ringbuffer_read(struct vmbus_channel *channel, memcpy(buffer, (const char *)desc + offset, packetlen); /* Advance ring index to next packet descriptor */ - __hv_pkt_iter_next(channel, desc, true); + __hv_pkt_iter_next(channel, desc); /* Notify host of update */ hv_pkt_iter_close(channel); @@ -422,22 +411,6 @@ static u32 hv_pkt_iter_avail(const struct hv_ring_buffer_info *rbi) return (rbi->ring_datasize - priv_read_loc) + write_loc; } -/* - * Get first vmbus packet without copying it out of the ring buffer - */ -struct vmpacket_descriptor *hv_pkt_iter_first_raw(struct vmbus_channel *channel) -{ - struct hv_ring_buffer_info *rbi = &channel->inbound; - - hv_debug_delay_test(channel, MESSAGE_DELAY); - - if (hv_pkt_iter_avail(rbi) < sizeof(struct vmpacket_descriptor)) - return NULL; - - return (struct vmpacket_descriptor *)(hv_get_ring_buffer(rbi) + rbi->priv_read_index); -} -EXPORT_SYMBOL_GPL(hv_pkt_iter_first_raw); - /* * Get first vmbus packet from ring buffer after read_index * @@ -446,49 +419,17 @@ EXPORT_SYMBOL_GPL(hv_pkt_iter_first_raw); struct vmpacket_descriptor *hv_pkt_iter_first(struct vmbus_channel *channel) { struct hv_ring_buffer_info *rbi = &channel->inbound; - struct vmpacket_descriptor *desc, *desc_copy; - u32 bytes_avail, pkt_len, pkt_offset; + struct vmpacket_descriptor *desc; - desc = hv_pkt_iter_first_raw(channel); - if (!desc) + hv_debug_delay_test(channel, MESSAGE_DELAY); + if (hv_pkt_iter_avail(rbi) < sizeof(struct vmpacket_descriptor)) return NULL; - bytes_avail = min(rbi->pkt_buffer_size, hv_pkt_iter_avail(rbi)); - - /* - * Ensure the compiler does not use references to incoming Hyper-V values (which - * could change at any moment) when reading local variables later in the code - */ - pkt_len = READ_ONCE(desc->len8) << 3; - pkt_offset = READ_ONCE(desc->offset8) << 3; - - /* - * If pkt_len is invalid, set it to the smaller of hv_pkt_iter_avail() and - * rbi->pkt_buffer_size - */ - if (pkt_len < sizeof(struct vmpacket_descriptor) || pkt_len > bytes_avail) - pkt_len = bytes_avail; - - /* - * If pkt_offset is invalid, arbitrarily set it to - * the size of vmpacket_descriptor - */ - if (pkt_offset < sizeof(struct vmpacket_descriptor) || pkt_offset > pkt_len) - pkt_offset = sizeof(struct vmpacket_descriptor); - - /* Copy the Hyper-V packet out of the ring buffer */ - desc_copy = (struct vmpacket_descriptor *)rbi->pkt_buffer; - memcpy(desc_copy, desc, pkt_len); - - /* - * Hyper-V could still change len8 and offset8 after the earlier read. - * Ensure that desc_copy has legal values for len8 and offset8 that - * are consistent with the copy we just made - */ - desc_copy->len8 = pkt_len >> 3; - desc_copy->offset8 = pkt_offset >> 3; + desc = hv_get_ring_buffer(rbi) + rbi->priv_read_index; + if (desc) + prefetch((char *)desc + (desc->len8 << 3)); - return desc_copy; + return desc; } EXPORT_SYMBOL_GPL(hv_pkt_iter_first); @@ -500,8 +441,7 @@ EXPORT_SYMBOL_GPL(hv_pkt_iter_first); */ struct vmpacket_descriptor * __hv_pkt_iter_next(struct vmbus_channel *channel, - const struct vmpacket_descriptor *desc, - bool copy) + const struct vmpacket_descriptor *desc) { struct hv_ring_buffer_info *rbi = &channel->inbound; u32 packetlen = desc->len8 << 3; @@ -514,7 +454,7 @@ __hv_pkt_iter_next(struct vmbus_channel *channel, rbi->priv_read_index -= dsize; /* more data? */ - return copy ? hv_pkt_iter_first(channel) : hv_pkt_iter_first_raw(channel); + return hv_pkt_iter_first(channel); } EXPORT_SYMBOL_GPL(__hv_pkt_iter_next); diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h index 7ea6936f86ef..2a87cfa27ac0 100644 --- a/drivers/net/hyperv/hyperv_net.h +++ b/drivers/net/hyperv/hyperv_net.h @@ -860,12 +860,9 @@ static inline u32 netvsc_rqstor_size(unsigned long ringbytes) ringbytes / NETVSC_MIN_IN_MSG_SIZE; } -#define NETVSC_MAX_XFER_PAGE_RANGES 375 #define NETVSC_XFER_HEADER_SIZE(rng_cnt) \ (offsetof(struct vmtransfer_page_packet_header, ranges) + \ (rng_cnt) * sizeof(struct vmtransfer_page_range)) -#define NETVSC_MAX_PKT_SIZE (NETVSC_XFER_HEADER_SIZE(NETVSC_MAX_XFER_PAGE_RANGES) + \ - sizeof(struct nvsp_message) + (sizeof(u32) * VRSS_SEND_TAB_SIZE)) struct multi_send_data { struct sk_buff *skb; /* skb containing the pkt */ diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c index 51005f2d4a82..2353623259f3 100644 --- a/drivers/net/hyperv/netvsc.c +++ b/drivers/net/hyperv/netvsc.c @@ -1544,8 +1544,6 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device, /* Open the channel */ device->channel->rqstor_size = netvsc_rqstor_size(netvsc_ring_bytes); - device->channel->max_pkt_size = NETVSC_MAX_PKT_SIZE; - ret = vmbus_open(device->channel, netvsc_ring_bytes, netvsc_ring_bytes, NULL, 0, netvsc_channel_cb, net_device->chan_table); diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c index 7e6dee2f02a4..598713c0d5a8 100644 --- a/drivers/net/hyperv/rndis_filter.c +++ b/drivers/net/hyperv/rndis_filter.c @@ -1174,8 +1174,6 @@ static void netvsc_sc_open(struct vmbus_channel *new_sc) nvchan->channel = new_sc; new_sc->rqstor_size = netvsc_rqstor_size(netvsc_ring_bytes); - new_sc->max_pkt_size = NETVSC_MAX_PKT_SIZE; - ret = vmbus_open(new_sc, netvsc_ring_bytes, netvsc_ring_bytes, NULL, 0, netvsc_channel_cb, nvchan); diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c index 7e59284dbf5b..2e4fa77445fd 100644 --- a/drivers/scsi/storvsc_drv.c +++ b/drivers/scsi/storvsc_drv.c @@ -414,14 +414,6 @@ static void storvsc_on_channel_callback(void *context); #define STORVSC_IDE_MAX_TARGETS 1 #define STORVSC_IDE_MAX_CHANNELS 1 -/* - * Upper bound on the size of a storvsc packet. vmscsi_size_delta is not - * included in the calculation because it is set after STORVSC_MAX_PKT_SIZE - * is used in storvsc_connect_to_vsp - */ -#define STORVSC_MAX_PKT_SIZE (sizeof(struct vmpacket_descriptor) +\ - sizeof(struct vstor_packet)) - struct storvsc_cmd_request { struct scsi_cmnd *cmd; @@ -706,7 +698,6 @@ static void handle_sc_creation(struct vmbus_channel *new_sc) return; memset(&props, 0, sizeof(struct vmstorage_channel_properties)); - new_sc->max_pkt_size = STORVSC_MAX_PKT_SIZE; /* * The size of vmbus_requestor is an upper bound on the number of requests @@ -1289,7 +1280,6 @@ static int storvsc_connect_to_vsp(struct hv_device *device, u32 ring_size, memset(&props, 0, sizeof(struct vmstorage_channel_properties)); - device->channel->max_pkt_size = STORVSC_MAX_PKT_SIZE; /* * The size of vmbus_requestor is an upper bound on the number of requests * that can be in-progress at any one time across all channels. diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 9dd22af1b7f6..f1d74dcf0353 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -181,10 +181,6 @@ struct hv_ring_buffer_info { * being freed while the ring buffer is being accessed. */ struct mutex ring_buffer_mutex; - - /* Buffer that holds a copy of an incoming host packet */ - void *pkt_buffer; - u32 pkt_buffer_size; }; @@ -792,8 +788,6 @@ struct vmbus_device { bool allowed_in_isolated; }; -#define VMBUS_DEFAULT_MAX_PKT_SIZE 4096 - struct vmbus_channel { struct list_head listentry; @@ -1016,9 +1010,6 @@ struct vmbus_channel { /* request/transaction ids for VMBus */ struct vmbus_requestor requestor; u32 rqstor_size; - - /* The max size of a packet on this channel */ - u32 max_pkt_size; }; u64 vmbus_next_request_id(struct vmbus_requestor *rqstor, u64 rqst_addr); @@ -1660,44 +1651,15 @@ static inline u32 hv_pkt_datalen(const struct vmpacket_descriptor *desc) } -struct vmpacket_descriptor * -hv_pkt_iter_first_raw(struct vmbus_channel *channel); - struct vmpacket_descriptor * hv_pkt_iter_first(struct vmbus_channel *channel); struct vmpacket_descriptor * __hv_pkt_iter_next(struct vmbus_channel *channel, - const struct vmpacket_descriptor *pkt, - bool copy); + const struct vmpacket_descriptor *pkt); void hv_pkt_iter_close(struct vmbus_channel *channel); -static inline struct vmpacket_descriptor * -hv_pkt_iter_next_pkt(struct vmbus_channel *channel, - const struct vmpacket_descriptor *pkt, - bool copy) -{ - struct vmpacket_descriptor *nxt; - - nxt = __hv_pkt_iter_next(channel, pkt, copy); - if (!nxt) - hv_pkt_iter_close(channel); - - return nxt; -} - -/* - * Get next packet descriptor without copying it out of the ring buffer - * If at end of list, return NULL and update host. - */ -static inline struct vmpacket_descriptor * -hv_pkt_iter_next_raw(struct vmbus_channel *channel, - const struct vmpacket_descriptor *pkt) -{ - return hv_pkt_iter_next_pkt(channel, pkt, false); -} - /* * Get next packet descriptor from iterator * If at end of list, return NULL and update host. @@ -1706,7 +1668,13 @@ static inline struct vmpacket_descriptor * hv_pkt_iter_next(struct vmbus_channel *channel, const struct vmpacket_descriptor *pkt) { - return hv_pkt_iter_next_pkt(channel, pkt, true); + struct vmpacket_descriptor *nxt; + + nxt = __hv_pkt_iter_next(channel, pkt); + if (!nxt) + hv_pkt_iter_close(channel); + + return nxt; } #define foreach_vmbus_pkt(pkt, channel) \ diff --git a/net/vmw_vsock/hyperv_transport.c b/net/vmw_vsock/hyperv_transport.c index cd8b7c1ca9f1..630b851f8150 100644 --- a/net/vmw_vsock/hyperv_transport.c +++ b/net/vmw_vsock/hyperv_transport.c @@ -600,7 +600,7 @@ static ssize_t hvs_stream_dequeue(struct vsock_sock *vsk, struct msghdr *msg, return -EOPNOTSUPP; if (need_refill) { - hvs->recv_desc = hv_pkt_iter_first_raw(hvs->chan); + hvs->recv_desc = hv_pkt_iter_first(hvs->chan); ret = hvs_update_recv_data(hvs); if (ret) return ret; @@ -614,7 +614,7 @@ static ssize_t hvs_stream_dequeue(struct vsock_sock *vsk, struct msghdr *msg, hvs->recv_data_len -= to_read; if (hvs->recv_data_len == 0) { - hvs->recv_desc = hv_pkt_iter_next_raw(hvs->chan, hvs->recv_desc); + hvs->recv_desc = hv_pkt_iter_next(hvs->chan, hvs->recv_desc); if (hvs->recv_desc) { ret = hvs_update_recv_data(hvs); if (ret) -- cgit v1.2.3 From 2347961b11d4079deace3c81dceed460c08a8fc1 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 28 Jan 2020 14:25:39 +0100 Subject: binfmt_misc: pass binfmt_misc flags to the interpreter It can be useful to the interpreter to know which flags are in use. For instance, knowing if the preserve-argv[0] is in use would allow to skip the pathname argument. This patch uses an unused auxiliary vector, AT_FLAGS, to add a flag to inform interpreter if the preserve-argv[0] is enabled. Note by Helge Deller: The real-world user of this patch is qemu-user, which needs to know if it has to preserve the argv[0]. See Debian bug #970460. Signed-off-by: Laurent Vivier Reviewed-by: YunQiang Su URL: http://bugs.debian.org/970460 Signed-off-by: Helge Deller --- fs/binfmt_elf.c | 5 ++++- fs/binfmt_elf_fdpic.c | 5 ++++- fs/binfmt_misc.c | 4 +++- include/linux/binfmts.h | 4 ++++ include/uapi/linux/binfmts.h | 4 ++++ 5 files changed, 19 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 950bc177238a..11fb68e68f65 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -186,6 +186,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, unsigned char k_rand_bytes[16]; int items; elf_addr_t *elf_info; + elf_addr_t flags = 0; int ei_index; const struct cred *cred = current_cred(); struct vm_area_struct *vma; @@ -260,7 +261,9 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr)); NEW_AUX_ENT(AT_PHNUM, exec->e_phnum); NEW_AUX_ENT(AT_BASE, interp_load_addr); - NEW_AUX_ENT(AT_FLAGS, 0); + if (bprm->interp_flags & BINPRM_FLAGS_PRESERVE_ARGV0) + flags |= AT_FLAGS_PRESERVE_ARGV0; + NEW_AUX_ENT(AT_FLAGS, flags); NEW_AUX_ENT(AT_ENTRY, e_entry); NEW_AUX_ENT(AT_UID, from_kuid_munged(cred->user_ns, cred->uid)); NEW_AUX_ENT(AT_EUID, from_kuid_munged(cred->user_ns, cred->euid)); diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index be4062b8ba75..7a78c91bc1c1 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -506,6 +506,7 @@ static int create_elf_fdpic_tables(struct linux_binprm *bprm, char __user *u_platform, *u_base_platform, *p; int loop; int nr; /* reset for each csp adjustment */ + unsigned long flags = 0; #ifdef CONFIG_MMU /* In some cases (e.g. Hyper-Threading), we want to avoid L1 evictions @@ -648,7 +649,9 @@ static int create_elf_fdpic_tables(struct linux_binprm *bprm, NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr)); NEW_AUX_ENT(AT_PHNUM, exec_params->hdr.e_phnum); NEW_AUX_ENT(AT_BASE, interp_params->elfhdr_addr); - NEW_AUX_ENT(AT_FLAGS, 0); + if (bprm->interp_flags & BINPRM_FLAGS_PRESERVE_ARGV0) + flags |= AT_FLAGS_PRESERVE_ARGV0; + NEW_AUX_ENT(AT_FLAGS, flags); NEW_AUX_ENT(AT_ENTRY, exec_params->entry_addr); NEW_AUX_ENT(AT_UID, (elf_addr_t) from_kuid_munged(cred->user_ns, cred->uid)); NEW_AUX_ENT(AT_EUID, (elf_addr_t) from_kuid_munged(cred->user_ns, cred->euid)); diff --git a/fs/binfmt_misc.c b/fs/binfmt_misc.c index 3880a82da1dc..c457334de43f 100644 --- a/fs/binfmt_misc.c +++ b/fs/binfmt_misc.c @@ -153,7 +153,9 @@ static int load_misc_binary(struct linux_binprm *bprm) if (bprm->interp_flags & BINPRM_FLAGS_PATH_INACCESSIBLE) goto ret; - if (!(fmt->flags & MISC_FMT_PRESERVE_ARGV0)) { + if (fmt->flags & MISC_FMT_PRESERVE_ARGV0) { + bprm->interp_flags |= BINPRM_FLAGS_PRESERVE_ARGV0; + } else { retval = remove_arg_zero(bprm); if (retval) goto ret; diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index 0571701ab1c5..0abd93efc181 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -73,6 +73,10 @@ struct linux_binprm { #define BINPRM_FLAGS_PATH_INACCESSIBLE_BIT 2 #define BINPRM_FLAGS_PATH_INACCESSIBLE (1 << BINPRM_FLAGS_PATH_INACCESSIBLE_BIT) +/* preserve argv0 for the interpreter */ +#define BINPRM_FLAGS_PRESERVE_ARGV0_BIT 3 +#define BINPRM_FLAGS_PRESERVE_ARGV0 (1 << BINPRM_FLAGS_PRESERVE_ARGV0_BIT) + /* Function parameter for binfmt->coredump */ struct coredump_params { const kernel_siginfo_t *siginfo; diff --git a/include/uapi/linux/binfmts.h b/include/uapi/linux/binfmts.h index 689025d9c185..c6f9450efc12 100644 --- a/include/uapi/linux/binfmts.h +++ b/include/uapi/linux/binfmts.h @@ -18,4 +18,8 @@ struct pt_regs; /* sizeof(linux_binprm->buf) */ #define BINPRM_BUF_SIZE 256 +/* preserve argv0 for the interpreter */ +#define AT_FLAGS_PRESERVE_ARGV0_BIT 0 +#define AT_FLAGS_PRESERVE_ARGV0 (1 << AT_FLAGS_PRESERVE_ARGV0_BIT) + #endif /* _UAPI_LINUX_BINFMTS_H */ -- cgit v1.2.3 From d0a0bbe7b0a181c58bd22d6942146cfa3ab9e49a Mon Sep 17 00:00:00 2001 From: Tong Zhang Date: Sun, 14 Feb 2021 18:43:08 -0500 Subject: atm: idt77252: fix build broken on amd64 idt77252 is broken and wont load on amd64 systems modprobe idt77252 shows the following idt77252_init: skb->cb is too small (48 < 56) Add packed attribute to struct idt77252_skb_prv and struct atm_skb_data so that the total size can be <= sizeof(skb->cb) Also convert runtime size check to buildtime size check in idt77252_init() Signed-off-by: Tong Zhang Signed-off-by: David S. Miller --- drivers/atm/idt77252.c | 11 +---------- drivers/atm/idt77252.h | 2 +- include/linux/atmdev.h | 2 +- 3 files changed, 3 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/atm/idt77252.c b/drivers/atm/idt77252.c index 5f0472c18bcb..0c13cac903de 100644 --- a/drivers/atm/idt77252.c +++ b/drivers/atm/idt77252.c @@ -3743,16 +3743,7 @@ static int __init idt77252_init(void) struct sk_buff *skb; printk("%s: at %p\n", __func__, idt77252_init); - - if (sizeof(skb->cb) < sizeof(struct atm_skb_data) + - sizeof(struct idt77252_skb_prv)) { - printk(KERN_ERR "%s: skb->cb is too small (%lu < %lu)\n", - __func__, (unsigned long) sizeof(skb->cb), - (unsigned long) sizeof(struct atm_skb_data) + - sizeof(struct idt77252_skb_prv)); - return -EIO; - } - + BUILD_BUG_ON(sizeof(skb->cb) < sizeof(struct idt77252_skb_prv) + sizeof(struct atm_skb_data)); return pci_register_driver(&idt77252_driver); } diff --git a/drivers/atm/idt77252.h b/drivers/atm/idt77252.h index 9339197d701c..b059d31364dd 100644 --- a/drivers/atm/idt77252.h +++ b/drivers/atm/idt77252.h @@ -789,7 +789,7 @@ struct idt77252_skb_prv { struct scqe tbd; /* Transmit Buffer Descriptor */ dma_addr_t paddr; /* DMA handle */ u32 pool; /* sb_pool handle */ -}; +} __packed; #define IDT77252_PRV_TBD(skb) \ (((struct idt77252_skb_prv *)(ATM_SKB(skb)+1))->tbd) diff --git a/include/linux/atmdev.h b/include/linux/atmdev.h index d7493016cd46..60cd25c0461b 100644 --- a/include/linux/atmdev.h +++ b/include/linux/atmdev.h @@ -207,7 +207,7 @@ struct atm_skb_data { struct atm_vcc *vcc; /* ATM VCC */ unsigned long atm_options; /* ATM layer options */ unsigned int acct_truesize; /* truesize accounted to vcc */ -}; +} __packed; #define VCC_HTABLE_SIZE 32 -- cgit v1.2.3 From 17d3a83afbbff34209d6c3636718fc1abe305ef8 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Fri, 12 Feb 2021 19:46:31 -0800 Subject: net: phy: broadcom: Remove unused flags We have a number of unused flags defined today and since we are scarce on space and may need to introduce new flags in the future remove and shift every existing flag down into a contiguous assignment. PHY_BCM_FLAGS_MODE_1000BX was only used internally for the BCM54616S PHY, so we allocate a driver private structure instead to store that flag instead of canibalizing one from phydev->dev_flags for that purpose. Signed-off-by: Florian Fainelli Reviewed-by: Vladimir Oltean Signed-off-by: David S. Miller --- drivers/net/phy/broadcom.c | 19 ++++++++++++++++--- include/linux/brcmphy.h | 21 ++++++++------------- 2 files changed, 24 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c index 4142f69c1530..3ce266ab521b 100644 --- a/drivers/net/phy/broadcom.c +++ b/drivers/net/phy/broadcom.c @@ -381,10 +381,21 @@ static int bcm5481_config_aneg(struct phy_device *phydev) return ret; } +struct bcm54616s_phy_priv { + bool mode_1000bx_en; +}; + static int bcm54616s_probe(struct phy_device *phydev) { + struct bcm54616s_phy_priv *priv; int val, intf_sel; + priv = devm_kzalloc(&phydev->mdio.dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + phydev->priv = priv; + val = bcm_phy_read_shadow(phydev, BCM54XX_SHD_MODE); if (val < 0) return val; @@ -407,7 +418,7 @@ static int bcm54616s_probe(struct phy_device *phydev) * 1000BASE-X configuration. */ if (!(val & BCM54616S_100FX_MODE)) - phydev->dev_flags |= PHY_BCM_FLAGS_MODE_1000BX; + priv->mode_1000bx_en = true; phydev->port = PORT_FIBRE; } @@ -417,10 +428,11 @@ static int bcm54616s_probe(struct phy_device *phydev) static int bcm54616s_config_aneg(struct phy_device *phydev) { + struct bcm54616s_phy_priv *priv = phydev->priv; int ret; /* Aneg firstly. */ - if (phydev->dev_flags & PHY_BCM_FLAGS_MODE_1000BX) + if (priv->mode_1000bx_en) ret = genphy_c37_config_aneg(phydev); else ret = genphy_config_aneg(phydev); @@ -433,9 +445,10 @@ static int bcm54616s_config_aneg(struct phy_device *phydev) static int bcm54616s_read_status(struct phy_device *phydev) { + struct bcm54616s_phy_priv *priv = phydev->priv; int err; - if (phydev->dev_flags & PHY_BCM_FLAGS_MODE_1000BX) + if (priv->mode_1000bx_en) err = genphy_c37_read_status(phydev); else err = genphy_read_status(phydev); diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index de9430d55c90..844dcfe789a2 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -61,19 +61,14 @@ #define PHY_BCM_OUI_5 0x03625e00 #define PHY_BCM_OUI_6 0xae025000 -#define PHY_BCM_FLAGS_MODE_COPPER 0x00000001 -#define PHY_BCM_FLAGS_MODE_1000BX 0x00000002 -#define PHY_BCM_FLAGS_INTF_SGMII 0x00000010 -#define PHY_BCM_FLAGS_INTF_XAUI 0x00000020 -#define PHY_BRCM_WIRESPEED_ENABLE 0x00000100 -#define PHY_BRCM_AUTO_PWRDWN_ENABLE 0x00000200 -#define PHY_BRCM_RX_REFCLK_UNUSED 0x00000400 -#define PHY_BRCM_STD_IBND_DISABLE 0x00000800 -#define PHY_BRCM_EXT_IBND_RX_ENABLE 0x00001000 -#define PHY_BRCM_EXT_IBND_TX_ENABLE 0x00002000 -#define PHY_BRCM_CLEAR_RGMII_MODE 0x00004000 -#define PHY_BRCM_DIS_TXCRXC_NOENRGY 0x00008000 -#define PHY_BRCM_EN_MASTER_MODE 0x00010000 +#define PHY_BRCM_AUTO_PWRDWN_ENABLE 0x00000001 +#define PHY_BRCM_RX_REFCLK_UNUSED 0x00000002 +#define PHY_BRCM_STD_IBND_DISABLE 0x00000004 +#define PHY_BRCM_EXT_IBND_RX_ENABLE 0x00000008 +#define PHY_BRCM_EXT_IBND_TX_ENABLE 0x00000010 +#define PHY_BRCM_CLEAR_RGMII_MODE 0x00000020 +#define PHY_BRCM_DIS_TXCRXC_NOENRGY 0x00000040 +#define PHY_BRCM_EN_MASTER_MODE 0x00000080 /* Broadcom BCM7xxx specific workarounds */ #define PHY_BRCM_7XXX_REV(x) (((x) >> 8) & 0xff) -- cgit v1.2.3 From 5d4358ede8ebe2e4ae03a633082f3ce21ec2df3e Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Fri, 12 Feb 2021 19:46:32 -0800 Subject: net: phy: broadcom: Allow BCM54210E to configure APD BCM54210E/BCM50212E has been verified to work correctly with the auto-power down configuration done by bcm54xx_adjust_rxrefclk(), add it to the list of PHYs working. While we are at it, provide an appropriate name for the bit we are changing which disables the RXC and TXC during auto-power down when there is no energy on the cable. Signed-off-by: Florian Fainelli Reviewed-by: Vladimir Oltean Signed-off-by: David S. Miller --- drivers/net/phy/broadcom.c | 8 +++++--- include/linux/brcmphy.h | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c index 3ce266ab521b..91fbd26c809e 100644 --- a/drivers/net/phy/broadcom.c +++ b/drivers/net/phy/broadcom.c @@ -193,6 +193,7 @@ static void bcm54xx_adjust_rxrefclk(struct phy_device *phydev) if (BRCM_PHY_MODEL(phydev) != PHY_ID_BCM57780 && BRCM_PHY_MODEL(phydev) != PHY_ID_BCM50610 && BRCM_PHY_MODEL(phydev) != PHY_ID_BCM50610M && + BRCM_PHY_MODEL(phydev) != PHY_ID_BCM54210E && BRCM_PHY_MODEL(phydev) != PHY_ID_BCM54810 && BRCM_PHY_MODEL(phydev) != PHY_ID_BCM54811) return; @@ -227,9 +228,10 @@ static void bcm54xx_adjust_rxrefclk(struct phy_device *phydev) val |= BCM54XX_SHD_SCR3_DLLAPD_DIS; if (phydev->dev_flags & PHY_BRCM_DIS_TXCRXC_NOENRGY) { - if (BRCM_PHY_MODEL(phydev) == PHY_ID_BCM54810 || - BRCM_PHY_MODEL(phydev) == PHY_ID_BCM54811) - val |= BCM54810_SHD_SCR3_TRDDAPD; + if (BRCM_PHY_MODEL(phydev) == PHY_ID_BCM54210E || + BRCM_PHY_MODEL(phydev) == PHY_ID_BCM54810 || + BRCM_PHY_MODEL(phydev) == PHY_ID_BCM54210E) + val |= BCM54XX_SHD_SCR3_RXCTXC_DIS; else val |= BCM54XX_SHD_SCR3_TRDDAPD; } diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index 844dcfe789a2..16597d3fa011 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -193,6 +193,7 @@ #define BCM54XX_SHD_SCR3_DEF_CLK125 0x0001 #define BCM54XX_SHD_SCR3_DLLAPD_DIS 0x0002 #define BCM54XX_SHD_SCR3_TRDDAPD 0x0004 +#define BCM54XX_SHD_SCR3_RXCTXC_DIS 0x0100 /* 01010: Auto Power-Down */ #define BCM54XX_SHD_APD 0x0a @@ -253,7 +254,6 @@ #define BCM54810_EXP_BROADREACH_LRE_MISC_CTL_EN (1 << 0) #define BCM54810_SHD_CLK_CTL 0x3 #define BCM54810_SHD_CLK_CTL_GTXCLK_EN (1 << 9) -#define BCM54810_SHD_SCR3_TRDDAPD 0x0100 /* BCM54612E Registers */ #define BCM54612E_EXP_SPARE0 (MII_BCM54XX_EXP_SEL_ETC + 0x34) -- cgit v1.2.3 From 93e8990c24bee30696c02e8f6aed043333491a25 Mon Sep 17 00:00:00 2001 From: Heiner Kallweit Date: Sun, 14 Feb 2021 15:16:23 +0100 Subject: net: phy: rename PHY_IGNORE_INTERRUPT to PHY_MAC_INTERRUPT Some internal PHY's have their events like link change reported by the MAC interrupt. We have PHY_IGNORE_INTERRUPT to deal with this scenario. I'm not too happy with this name. We don't ignore interrupts, typically there is no interrupt exposed at a PHY level. So let's rename it to PHY_MAC_INTERRUPT. This is in line with phy_mac_interrupt(), which is called from the MAC interrupt handler to handle PHY events. Signed-off-by: Heiner Kallweit Reviewed-by: Andrew Lunn Acked-by: Florian Fainelli Reviewed-by: Russell King Signed-off-by: David S. Miller --- Documentation/networking/phy.rst | 2 +- drivers/net/ethernet/broadcom/genet/bcmmii.c | 2 +- drivers/net/ethernet/realtek/r8169_main.c | 2 +- drivers/net/ethernet/samsung/sxgbe/sxgbe_mdio.c | 4 ++-- drivers/net/mdio/mdio-moxart.c | 4 ++-- drivers/net/phy/icplus.c | 2 +- drivers/net/phy/phy.c | 2 +- drivers/net/phy/phy_device.c | 4 ++-- include/linux/phy.h | 10 +++++----- 9 files changed, 16 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/phy.rst b/Documentation/networking/phy.rst index 399f17976a6c..70136cc9e25e 100644 --- a/Documentation/networking/phy.rst +++ b/Documentation/networking/phy.rst @@ -216,7 +216,7 @@ put into an unsupported state. Lastly, once the controller is ready to handle network traffic, you call phy_start(phydev). This tells the PAL that you are ready, and configures the PHY to connect to the network. If the MAC interrupt of your network driver -also handles PHY status changes, just set phydev->irq to PHY_IGNORE_INTERRUPT +also handles PHY status changes, just set phydev->irq to PHY_MAC_INTERRUPT before you call phy_start and use phy_mac_interrupt() from the network driver. If you don't want to use interrupts, set phydev->irq to PHY_POLL. phy_start() enables the PHY interrupts (if applicable) and starts the diff --git a/drivers/net/ethernet/broadcom/genet/bcmmii.c b/drivers/net/ethernet/broadcom/genet/bcmmii.c index 17f997ef950f..5335244e4577 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmmii.c +++ b/drivers/net/ethernet/broadcom/genet/bcmmii.c @@ -359,7 +359,7 @@ int bcmgenet_mii_probe(struct net_device *dev) * those versions of GENET. */ if (priv->internal_phy && !GENET_IS_V5(priv)) - dev->phydev->irq = PHY_IGNORE_INTERRUPT; + dev->phydev->irq = PHY_MAC_INTERRUPT; return 0; } diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 44a438bad153..cbc30df4e08a 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -5056,7 +5056,7 @@ static int r8169_mdio_register(struct rtl8169_private *tp) new_bus->name = "r8169"; new_bus->priv = tp; new_bus->parent = &pdev->dev; - new_bus->irq[0] = PHY_IGNORE_INTERRUPT; + new_bus->irq[0] = PHY_MAC_INTERRUPT; snprintf(new_bus->id, MII_BUS_ID_SIZE, "r8169-%x", pci_dev_id(pdev)); new_bus->read = r8169_mdio_read_reg; diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_mdio.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_mdio.c index b1e7f7ab281c..fceb6d637235 100644 --- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_mdio.c +++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_mdio.c @@ -203,8 +203,8 @@ int sxgbe_mdio_register(struct net_device *ndev) case PHY_POLL: irq_str = "POLL"; break; - case PHY_IGNORE_INTERRUPT: - irq_str = "IGNORE"; + case PHY_MAC_INTERRUPT: + irq_str = "MAC"; break; default: sprintf(irq_num, "%d", phy->irq); diff --git a/drivers/net/mdio/mdio-moxart.c b/drivers/net/mdio/mdio-moxart.c index b72c6d185175..f0cff584e176 100644 --- a/drivers/net/mdio/mdio-moxart.c +++ b/drivers/net/mdio/mdio-moxart.c @@ -125,7 +125,7 @@ static int moxart_mdio_probe(struct platform_device *pdev) snprintf(bus->id, MII_BUS_ID_SIZE, "%s-%d-mii", pdev->name, pdev->id); bus->parent = &pdev->dev; - /* Setting PHY_IGNORE_INTERRUPT here even if it has no effect, + /* Setting PHY_MAC_INTERRUPT here even if it has no effect, * of_mdiobus_register() sets these PHY_POLL. * Ideally, the interrupt from MAC controller could be used to * detect link state changes, not polling, i.e. if there was @@ -133,7 +133,7 @@ static int moxart_mdio_probe(struct platform_device *pdev) * interrupt handled in ethernet drivercode. */ for (i = 0; i < PHY_MAX_ADDR; i++) - bus->irq[i] = PHY_IGNORE_INTERRUPT; + bus->irq[i] = PHY_MAC_INTERRUPT; data = bus->priv; data->base = devm_platform_ioremap_resource(pdev, 0); diff --git a/drivers/net/phy/icplus.c b/drivers/net/phy/icplus.c index 4e15d4d02488..3e431737c1ba 100644 --- a/drivers/net/phy/icplus.c +++ b/drivers/net/phy/icplus.c @@ -188,7 +188,7 @@ static int ip175c_read_status(struct phy_device *phydev) genphy_read_status(phydev); else /* Don't need to read status for switch ports */ - phydev->irq = PHY_IGNORE_INTERRUPT; + phydev->irq = PHY_MAC_INTERRUPT; return 0; } diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index fdb914b5b857..1be07e45d314 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -1145,7 +1145,7 @@ void phy_state_machine(struct work_struct *work) } /* Only re-schedule a PHY state machine change if we are polling the - * PHY, if PHY_IGNORE_INTERRUPT is set, then we will be moving + * PHY, if PHY_MAC_INTERRUPT is set, then we will be moving * between states from phy_mac_interrupt(). * * In state PHY_HALTED the PHY gets suspended, so rescheduling the diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 30a20a29ae05..05261698bf74 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1167,8 +1167,8 @@ char *phy_attached_info_irq(struct phy_device *phydev) case PHY_POLL: irq_str = "POLL"; break; - case PHY_IGNORE_INTERRUPT: - irq_str = "IGNORE"; + case PHY_MAC_INTERRUPT: + irq_str = "MAC"; break; default: snprintf(irq_num, sizeof(irq_num), "%d", phydev->irq); diff --git a/include/linux/phy.h b/include/linux/phy.h index c130788306c8..5d7c4084ade9 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -71,11 +71,11 @@ extern const int phy_10gbit_features_array[1]; /* * Set phydev->irq to PHY_POLL if interrupts are not supported, - * or not desired for this PHY. Set to PHY_IGNORE_INTERRUPT if - * the attached driver handles the interrupt + * or not desired for this PHY. Set to PHY_MAC_INTERRUPT if + * the attached MAC driver handles the interrupt */ #define PHY_POLL -1 -#define PHY_IGNORE_INTERRUPT -2 +#define PHY_MAC_INTERRUPT -2 #define PHY_IS_INTERNAL 0x00000001 #define PHY_RST_AFTER_CLK_EN 0x00000002 @@ -1202,11 +1202,11 @@ static inline int phy_clear_bits_mmd(struct phy_device *phydev, int devad, * @phydev: the phy_device struct * * NOTE: must be kept in sync with addition/removal of PHY_POLL and - * PHY_IGNORE_INTERRUPT + * PHY_MAC_INTERRUPT */ static inline bool phy_interrupt_is_valid(struct phy_device *phydev) { - return phydev->irq != PHY_POLL && phydev->irq != PHY_IGNORE_INTERRUPT; + return phydev->irq != PHY_POLL && phydev->irq != PHY_MAC_INTERRUPT; } /** -- cgit v1.2.3 From a6a217dddcd544f6b75f0e2a60b6e84c1d494b7e Mon Sep 17 00:00:00 2001 From: Aharon Landau Date: Tue, 9 Feb 2021 15:11:06 +0200 Subject: net/mlx5: Add new timestamp mode bits These fields declare which timestamp mode is supported by the device per RQ/SQ/QP. In addition add the ts_format field to the select the mode for RQ/SQ/QP. Link: https://lore.kernel.org/r/20210209131107.698833-2-leon@kernel.org Signed-off-by: Aharon Landau Signed-off-by: Maor Gottlieb Signed-off-by: Leon Romanovsky --- include/linux/mlx5/mlx5_ifc.h | 54 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index cf692fc17f41..436d6f421dfd 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -932,11 +932,18 @@ struct mlx5_ifc_per_protocol_networking_offload_caps_bits { u8 reserved_at_200[0x600]; }; +enum { + MLX5_QP_TIMESTAMP_FORMAT_CAP_FREE_RUNNING = 0x0, + MLX5_QP_TIMESTAMP_FORMAT_CAP_REAL_TIME = 0x1, + MLX5_QP_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME = 0x2, +}; + struct mlx5_ifc_roce_cap_bits { u8 roce_apm[0x1]; u8 reserved_at_1[0x3]; u8 sw_r_roce_src_udp_port[0x1]; - u8 reserved_at_5[0x1b]; + u8 reserved_at_5[0x19]; + u8 qp_ts_format[0x2]; u8 reserved_at_20[0x60]; @@ -1253,6 +1260,18 @@ enum { MLX5_STEERING_FORMAT_CONNECTX_6DX = 1, }; +enum { + MLX5_SQ_TIMESTAMP_FORMAT_CAP_FREE_RUNNING = 0x0, + MLX5_SQ_TIMESTAMP_FORMAT_CAP_REAL_TIME = 0x1, + MLX5_SQ_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME = 0x2, +}; + +enum { + MLX5_RQ_TIMESTAMP_FORMAT_CAP_FREE_RUNNING = 0x0, + MLX5_RQ_TIMESTAMP_FORMAT_CAP_REAL_TIME = 0x1, + MLX5_RQ_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME = 0x2, +}; + struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_0[0x1f]; u8 vhca_resource_manager[0x1]; @@ -1564,7 +1583,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 general_obj_types[0x40]; - u8 reserved_at_440[0x4]; + u8 sq_ts_format[0x2]; + u8 rq_ts_format[0x2]; u8 steering_format_version[0x4]; u8 create_qp_start_hint[0x18]; @@ -2868,6 +2888,12 @@ enum { MLX5_QPC_CS_RES_UP_TO_64B = 0x2, }; +enum { + MLX5_QPC_TIMESTAMP_FORMAT_FREE_RUNNING = 0x0, + MLX5_QPC_TIMESTAMP_FORMAT_DEFAULT = 0x1, + MLX5_QPC_TIMESTAMP_FORMAT_REAL_TIME = 0x2, +}; + struct mlx5_ifc_qpc_bits { u8 state[0x4]; u8 lag_tx_port_affinity[0x4]; @@ -2896,7 +2922,9 @@ struct mlx5_ifc_qpc_bits { u8 log_rq_stride[0x3]; u8 no_sq[0x1]; u8 log_sq_size[0x4]; - u8 reserved_at_55[0x6]; + u8 reserved_at_55[0x3]; + u8 ts_format[0x2]; + u8 reserved_at_5a[0x1]; u8 rlky[0x1]; u8 ulp_stateless_offload_mode[0x4]; @@ -3312,6 +3340,12 @@ enum { MLX5_SQC_STATE_ERR = 0x3, }; +enum { + MLX5_SQC_TIMESTAMP_FORMAT_FREE_RUNNING = 0x0, + MLX5_SQC_TIMESTAMP_FORMAT_DEFAULT = 0x1, + MLX5_SQC_TIMESTAMP_FORMAT_REAL_TIME = 0x2, +}; + struct mlx5_ifc_sqc_bits { u8 rlky[0x1]; u8 cd_master[0x1]; @@ -3323,7 +3357,9 @@ struct mlx5_ifc_sqc_bits { u8 reg_umr[0x1]; u8 allow_swp[0x1]; u8 hairpin[0x1]; - u8 reserved_at_f[0x11]; + u8 reserved_at_f[0xb]; + u8 ts_format[0x2]; + u8 reserved_at_1c[0x4]; u8 reserved_at_20[0x8]; u8 user_index[0x18]; @@ -3414,6 +3450,12 @@ enum { MLX5_RQC_STATE_ERR = 0x3, }; +enum { + MLX5_RQC_TIMESTAMP_FORMAT_FREE_RUNNING = 0x0, + MLX5_RQC_TIMESTAMP_FORMAT_DEFAULT = 0x1, + MLX5_RQC_TIMESTAMP_FORMAT_REAL_TIME = 0x2, +}; + struct mlx5_ifc_rqc_bits { u8 rlky[0x1]; u8 delay_drop_en[0x1]; @@ -3424,7 +3466,9 @@ struct mlx5_ifc_rqc_bits { u8 reserved_at_c[0x1]; u8 flush_in_error_en[0x1]; u8 hairpin[0x1]; - u8 reserved_at_f[0x11]; + u8 reserved_at_f[0xb]; + u8 ts_format[0x2]; + u8 reserved_at_1c[0x4]; u8 reserved_at_20[0x8]; u8 user_index[0x18]; -- cgit v1.2.3 From ae02d41551d6f2a035d3e63ce4415e1b2ba3a7e6 Mon Sep 17 00:00:00 2001 From: Eran Ben Elisha Date: Fri, 12 Feb 2021 14:30:38 -0800 Subject: net/mlx5: Add register layout to support real-time time-stamp Add needed structure layouts and defines for MTUTC (Management UTC) register. MTUTC will be used for cyc2time HW translation. In addition, add cyc2time modify capability bit and init segment HCA real time address. Finally, add capability bits indicating which time-stamping format is supported per SQ and RQ. Add ts_format in the queue's context layout to allow configuration. Signed-off-by: Eran Ben Elisha Signed-off-by: Aya Levin Reviewed-by: Moshe Shemesh Signed-off-by: Saeed Mahameed --- include/linux/mlx5/device.h | 5 ++++- include/linux/mlx5/driver.h | 1 + include/linux/mlx5/mlx5_ifc.h | 30 ++++++++++++++++++++++++++++-- 3 files changed, 33 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index f1de49d64a98..fd694add6ff0 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -578,7 +578,10 @@ struct mlx5_init_seg { __be32 internal_timer_l; __be32 rsvd3[2]; __be32 health_counter; - __be32 rsvd4[1019]; + __be32 rsvd4[11]; + __be32 real_time_h; + __be32 real_time_l; + __be32 rsvd5[1006]; __be64 ieee1588_clk; __be32 ieee1588_clk_type; __be32 clr_intx; diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index f93bfe7473aa..2eb7755143d7 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -143,6 +143,7 @@ enum { MLX5_REG_MPCNT = 0x9051, MLX5_REG_MTPPS = 0x9053, MLX5_REG_MTPPSE = 0x9054, + MLX5_REG_MTUTC = 0x9055, MLX5_REG_MPEGC = 0x9056, MLX5_REG_MCQS = 0x9060, MLX5_REG_MCQI = 0x9061, diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 436d6f421dfd..7059fcdf54a3 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -9150,6 +9150,28 @@ struct mlx5_ifc_mpegc_reg_bits { u8 reserved_at_60[0x100]; }; +enum { + MLX5_MTUTC_OPERATION_SET_TIME_IMMEDIATE = 0x1, + MLX5_MTUTC_OPERATION_ADJUST_TIME = 0x2, + MLX5_MTUTC_OPERATION_ADJUST_FREQ_UTC = 0x3, +}; + +struct mlx5_ifc_mtutc_reg_bits { + u8 reserved_at_0[0x1c]; + u8 operation[0x4]; + + u8 freq_adjustment[0x20]; + + u8 reserved_at_40[0x40]; + + u8 utc_sec[0x20]; + + u8 reserved_at_a0[0x2]; + u8 utc_nsec[0x1e]; + + u8 time_adjustment[0x20]; +}; + struct mlx5_ifc_pcam_enhanced_features_bits { u8 reserved_at_0[0x68]; u8 fec_50G_per_lane_in_pplm[0x1]; @@ -9208,7 +9230,9 @@ struct mlx5_ifc_pcam_reg_bits { }; struct mlx5_ifc_mcam_enhanced_features_bits { - u8 reserved_at_0[0x6e]; + u8 reserved_at_0[0x6b]; + u8 ptpcyc2realtime_modify[0x1]; + u8 reserved_at_6c[0x2]; u8 pci_status_and_power[0x1]; u8 reserved_at_6f[0x5]; u8 mark_tx_action_cnp[0x1]; @@ -9231,7 +9255,8 @@ struct mlx5_ifc_mcam_access_reg_bits { u8 regs_95_to_87[0x9]; u8 mpegc[0x1]; - u8 regs_85_to_68[0x12]; + u8 mtutc[0x1]; + u8 regs_84_to_68[0x11]; u8 tracer_registers[0x4]; u8 regs_63_to_32[0x20]; @@ -9964,6 +9989,7 @@ union mlx5_ifc_ports_control_registers_document_bits { struct mlx5_ifc_mcda_reg_bits mcda_reg; struct mlx5_ifc_mirc_reg_bits mirc_reg; struct mlx5_ifc_mfrl_reg_bits mfrl_reg; + struct mlx5_ifc_mtutc_reg_bits mtutc_reg; u8 reserved_at_0[0x60e0]; }; -- cgit v1.2.3 From d6f3dc8f509ce6288e2537eb4b0614ef444fd84a Mon Sep 17 00:00:00 2001 From: Eran Ben Elisha Date: Fri, 12 Feb 2021 14:30:40 -0800 Subject: net/mlx5: Move all internal timer metadata into a dedicated struct Internal timer mode (SW clock) requires some PTP clock related metadata structs. Real time mode (HW clock) will not need these metadata structs. This separation emphasize the different interfaces for HW clock and SW clock. Signed-off-by: Eran Ben Elisha Signed-off-by: Aya Levin Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/lib/clock.c | 107 ++++++++++++--------- .../net/ethernet/mellanox/mlx5/core/lib/clock.h | 3 +- include/linux/mlx5/driver.h | 12 ++- 3 files changed, 71 insertions(+), 51 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c index aaf7d837a967..dcacaa451b53 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c @@ -89,7 +89,8 @@ static u64 mlx5_read_internal_timer(struct mlx5_core_dev *dev, static u64 read_internal_timer(const struct cyclecounter *cc) { - struct mlx5_clock *clock = container_of(cc, struct mlx5_clock, cycles); + struct mlx5_timer *timer = container_of(cc, struct mlx5_timer, cycles); + struct mlx5_clock *clock = container_of(timer, struct mlx5_clock, timer); struct mlx5_core_dev *mdev = container_of(clock, struct mlx5_core_dev, clock); @@ -100,6 +101,7 @@ static void mlx5_update_clock_info_page(struct mlx5_core_dev *mdev) { struct mlx5_ib_clock_info *clock_info = mdev->clock_info; struct mlx5_clock *clock = &mdev->clock; + struct mlx5_timer *timer; u32 sign; if (!clock_info) @@ -109,10 +111,11 @@ static void mlx5_update_clock_info_page(struct mlx5_core_dev *mdev) smp_store_mb(clock_info->sign, sign | MLX5_IB_CLOCK_INFO_KERNEL_UPDATING); - clock_info->cycles = clock->tc.cycle_last; - clock_info->mult = clock->cycles.mult; - clock_info->nsec = clock->tc.nsec; - clock_info->frac = clock->tc.frac; + timer = &clock->timer; + clock_info->cycles = timer->tc.cycle_last; + clock_info->mult = timer->cycles.mult; + clock_info->nsec = timer->tc.nsec; + clock_info->frac = timer->tc.frac; smp_store_release(&clock_info->sign, sign + MLX5_IB_CLOCK_INFO_KERNEL_UPDATING * 2); @@ -151,28 +154,32 @@ static void mlx5_timestamp_overflow(struct work_struct *work) { struct delayed_work *dwork = to_delayed_work(work); struct mlx5_core_dev *mdev; + struct mlx5_timer *timer; struct mlx5_clock *clock; unsigned long flags; - clock = container_of(dwork, struct mlx5_clock, overflow_work); + timer = container_of(dwork, struct mlx5_timer, overflow_work); + clock = container_of(timer, struct mlx5_clock, timer); mdev = container_of(clock, struct mlx5_core_dev, clock); + write_seqlock_irqsave(&clock->lock, flags); - timecounter_read(&clock->tc); + timecounter_read(&timer->tc); mlx5_update_clock_info_page(mdev); write_sequnlock_irqrestore(&clock->lock, flags); - schedule_delayed_work(&clock->overflow_work, clock->overflow_period); + schedule_delayed_work(&timer->overflow_work, timer->overflow_period); } static int mlx5_ptp_settime(struct ptp_clock_info *ptp, const struct timespec64 *ts) { struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, ptp_info); + struct mlx5_timer *timer = &clock->timer; u64 ns = timespec64_to_ns(ts); struct mlx5_core_dev *mdev; unsigned long flags; mdev = container_of(clock, struct mlx5_core_dev, clock); write_seqlock_irqsave(&clock->lock, flags); - timecounter_init(&clock->tc, &clock->cycles, ns); + timecounter_init(&timer->tc, &timer->cycles, ns); mlx5_update_clock_info_page(mdev); write_sequnlock_irqrestore(&clock->lock, flags); @@ -183,6 +190,7 @@ static int mlx5_ptp_gettimex(struct ptp_clock_info *ptp, struct timespec64 *ts, struct ptp_system_timestamp *sts) { struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, ptp_info); + struct mlx5_timer *timer = &clock->timer; struct mlx5_core_dev *mdev; unsigned long flags; u64 cycles, ns; @@ -190,7 +198,7 @@ static int mlx5_ptp_gettimex(struct ptp_clock_info *ptp, struct timespec64 *ts, mdev = container_of(clock, struct mlx5_core_dev, clock); write_seqlock_irqsave(&clock->lock, flags); cycles = mlx5_read_internal_timer(mdev, sts); - ns = timecounter_cyc2time(&clock->tc, cycles); + ns = timecounter_cyc2time(&timer->tc, cycles); write_sequnlock_irqrestore(&clock->lock, flags); *ts = ns_to_timespec64(ns); @@ -201,12 +209,13 @@ static int mlx5_ptp_gettimex(struct ptp_clock_info *ptp, struct timespec64 *ts, static int mlx5_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) { struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, ptp_info); + struct mlx5_timer *timer = &clock->timer; struct mlx5_core_dev *mdev; unsigned long flags; mdev = container_of(clock, struct mlx5_core_dev, clock); write_seqlock_irqsave(&clock->lock, flags); - timecounter_adjtime(&clock->tc, delta); + timecounter_adjtime(&timer->tc, delta); mlx5_update_clock_info_page(mdev); write_sequnlock_irqrestore(&clock->lock, flags); @@ -216,27 +225,27 @@ static int mlx5_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) static int mlx5_ptp_adjfreq(struct ptp_clock_info *ptp, s32 delta) { struct mlx5_clock *clock = container_of(ptp, struct mlx5_clock, ptp_info); + struct mlx5_timer *timer = &clock->timer; struct mlx5_core_dev *mdev; unsigned long flags; int neg_adj = 0; u32 diff; u64 adj; - if (delta < 0) { neg_adj = 1; delta = -delta; } - adj = clock->nominal_c_mult; + adj = timer->nominal_c_mult; adj *= delta; diff = div_u64(adj, 1000000000ULL); mdev = container_of(clock, struct mlx5_core_dev, clock); write_seqlock_irqsave(&clock->lock, flags); - timecounter_read(&clock->tc); - clock->cycles.mult = neg_adj ? clock->nominal_c_mult - diff : - clock->nominal_c_mult + diff; + timecounter_read(&timer->tc); + timer->cycles.mult = neg_adj ? timer->nominal_c_mult - diff : + timer->nominal_c_mult + diff; mlx5_update_clock_info_page(mdev); write_sequnlock_irqrestore(&clock->lock, flags); @@ -313,6 +322,7 @@ static int mlx5_perout_configure(struct ptp_clock_info *ptp, container_of(ptp, struct mlx5_clock, ptp_info); struct mlx5_core_dev *mdev = container_of(clock, struct mlx5_core_dev, clock); + struct mlx5_timer *timer = &clock->timer; u32 in[MLX5_ST_SZ_DW(mtpps_reg)] = {0}; u64 nsec_now, nsec_delta, time_stamp = 0; u64 cycles_now, cycles_delta; @@ -355,10 +365,10 @@ static int mlx5_perout_configure(struct ptp_clock_info *ptp, ns = timespec64_to_ns(&ts); cycles_now = mlx5_read_internal_timer(mdev, NULL); write_seqlock_irqsave(&clock->lock, flags); - nsec_now = timecounter_cyc2time(&clock->tc, cycles_now); + nsec_now = timecounter_cyc2time(&timer->tc, cycles_now); nsec_delta = ns - nsec_now; - cycles_delta = div64_u64(nsec_delta << clock->cycles.shift, - clock->cycles.mult); + cycles_delta = div64_u64(nsec_delta << timer->cycles.shift, + timer->cycles.mult); write_sequnlock_irqrestore(&clock->lock, flags); time_stamp = cycles_now + cycles_delta; field_select = MLX5_MTPPS_FS_PIN_MODE | @@ -541,6 +551,7 @@ static int mlx5_pps_event(struct notifier_block *nb, unsigned long type, void *data) { struct mlx5_clock *clock = mlx5_nb_cof(nb, struct mlx5_clock, pps_nb); + struct mlx5_timer *timer = &clock->timer; struct ptp_clock_event ptp_event; u64 cycles_now, cycles_delta; u64 nsec_now, nsec_delta, ns; @@ -575,10 +586,10 @@ static int mlx5_pps_event(struct notifier_block *nb, ts.tv_nsec = 0; ns = timespec64_to_ns(&ts); write_seqlock_irqsave(&clock->lock, flags); - nsec_now = timecounter_cyc2time(&clock->tc, cycles_now); + nsec_now = timecounter_cyc2time(&timer->tc, cycles_now); nsec_delta = ns - nsec_now; - cycles_delta = div64_u64(nsec_delta << clock->cycles.shift, - clock->cycles.mult); + cycles_delta = div64_u64(nsec_delta << timer->cycles.shift, + timer->cycles.mult); clock->pps_info.start[pin] = cycles_now + cycles_delta; write_sequnlock_irqrestore(&clock->lock, flags); schedule_work(&clock->pps_info.out_work); @@ -594,17 +605,18 @@ static int mlx5_pps_event(struct notifier_block *nb, static void mlx5_timecounter_init(struct mlx5_core_dev *mdev) { struct mlx5_clock *clock = &mdev->clock; + struct mlx5_timer *timer = &clock->timer; u32 dev_freq; dev_freq = MLX5_CAP_GEN(mdev, device_frequency_khz); - clock->cycles.read = read_internal_timer; - clock->cycles.shift = MLX5_CYCLES_SHIFT; - clock->cycles.mult = clocksource_khz2mult(dev_freq, - clock->cycles.shift); - clock->nominal_c_mult = clock->cycles.mult; - clock->cycles.mask = CLOCKSOURCE_MASK(41); - - timecounter_init(&clock->tc, &clock->cycles, + timer->cycles.read = read_internal_timer; + timer->cycles.shift = MLX5_CYCLES_SHIFT; + timer->cycles.mult = clocksource_khz2mult(dev_freq, + timer->cycles.shift); + timer->nominal_c_mult = timer->cycles.mult; + timer->cycles.mask = CLOCKSOURCE_MASK(41); + + timecounter_init(&timer->tc, &timer->cycles, ktime_to_ns(ktime_get_real())); } @@ -612,6 +624,7 @@ static void mlx5_init_overflow_period(struct mlx5_clock *clock) { struct mlx5_core_dev *mdev = container_of(clock, struct mlx5_core_dev, clock); struct mlx5_ib_clock_info *clock_info = mdev->clock_info; + struct mlx5_timer *timer = &clock->timer; u64 overflow_cycles; u64 frac = 0; u64 ns; @@ -623,29 +636,30 @@ static void mlx5_init_overflow_period(struct mlx5_clock *clock) * multiplied by clock multiplier where the result doesn't exceed * 64bits. */ - overflow_cycles = div64_u64(~0ULL >> 1, clock->cycles.mult); - overflow_cycles = min(overflow_cycles, div_u64(clock->cycles.mask, 3)); + overflow_cycles = div64_u64(~0ULL >> 1, timer->cycles.mult); + overflow_cycles = min(overflow_cycles, div_u64(timer->cycles.mask, 3)); - ns = cyclecounter_cyc2ns(&clock->cycles, overflow_cycles, + ns = cyclecounter_cyc2ns(&timer->cycles, overflow_cycles, frac, &frac); do_div(ns, NSEC_PER_SEC / HZ); - clock->overflow_period = ns; + timer->overflow_period = ns; - INIT_DELAYED_WORK(&clock->overflow_work, mlx5_timestamp_overflow); - if (clock->overflow_period) - schedule_delayed_work(&clock->overflow_work, 0); + INIT_DELAYED_WORK(&timer->overflow_work, mlx5_timestamp_overflow); + if (timer->overflow_period) + schedule_delayed_work(&timer->overflow_work, 0); else mlx5_core_warn(mdev, "invalid overflow period, overflow_work is not scheduled\n"); if (clock_info) - clock_info->overflow_period = clock->overflow_period; + clock_info->overflow_period = timer->overflow_period; } static void mlx5_init_clock_info(struct mlx5_core_dev *mdev) { struct mlx5_clock *clock = &mdev->clock; struct mlx5_ib_clock_info *info; + struct mlx5_timer *timer; mdev->clock_info = (struct mlx5_ib_clock_info *)get_zeroed_page(GFP_KERNEL); if (!mdev->clock_info) { @@ -654,13 +668,14 @@ static void mlx5_init_clock_info(struct mlx5_core_dev *mdev) } info = mdev->clock_info; - - info->nsec = clock->tc.nsec; - info->cycles = clock->tc.cycle_last; - info->mask = clock->cycles.mask; - info->mult = clock->nominal_c_mult; - info->shift = clock->cycles.shift; - info->frac = clock->tc.frac; + timer = &clock->timer; + + info->nsec = timer->tc.nsec; + info->cycles = timer->tc.cycle_last; + info->mask = timer->cycles.mask; + info->mult = timer->nominal_c_mult; + info->shift = timer->cycles.shift; + info->frac = timer->tc.frac; } void mlx5_init_clock(struct mlx5_core_dev *mdev) @@ -714,7 +729,7 @@ void mlx5_cleanup_clock(struct mlx5_core_dev *mdev) } cancel_work_sync(&clock->pps_info.out_work); - cancel_delayed_work_sync(&clock->overflow_work); + cancel_delayed_work_sync(&clock->timer.overflow_work); if (mdev->clock_info) { free_page((unsigned long)mdev->clock_info); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.h index 31600924bdc3..6e8804ebc773 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.h @@ -45,12 +45,13 @@ static inline int mlx5_clock_get_ptp_index(struct mlx5_core_dev *mdev) static inline ktime_t mlx5_timecounter_cyc2time(struct mlx5_clock *clock, u64 timestamp) { + struct mlx5_timer *timer = &clock->timer; unsigned int seq; u64 nsec; do { seq = read_seqbegin(&clock->lock); - nsec = timecounter_cyc2time(&clock->tc, timestamp); + nsec = timecounter_cyc2time(&timer->tc, timestamp); } while (read_seqretry(&clock->lock, seq)); return ns_to_ktime(nsec); diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 2eb7755143d7..2f4d6182df1b 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -662,18 +662,22 @@ struct mlx5_pps { u8 enabled; }; -struct mlx5_clock { - struct mlx5_nb pps_nb; - seqlock_t lock; +struct mlx5_timer { struct cyclecounter cycles; struct timecounter tc; - struct hwtstamp_config hwtstamp_config; u32 nominal_c_mult; unsigned long overflow_period; struct delayed_work overflow_work; +}; + +struct mlx5_clock { + struct mlx5_nb pps_nb; + seqlock_t lock; + struct hwtstamp_config hwtstamp_config; struct ptp_clock *ptp; struct ptp_clock_info ptp_info; struct mlx5_pps pps_info; + struct mlx5_timer timer; }; struct mlx5_dm; -- cgit v1.2.3 From 32aeba1f7a98b0c69d4a5704a7d9cea42ba856ba Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 16 Feb 2021 11:08:37 -0800 Subject: tg3: Remove unused PHY_BRCM flags The tg3 driver tried to communicate towards the PHY driver whether it wanted RGMII in-band signaling enabled or disabled however there is nothing that looks at those flags in drivers/net/phy/broadcom.c so this does do not anything. Suggested-by: Vladimir Oltean Signed-off-by: Florian Fainelli Signed-off-by: David S. Miller --- drivers/net/ethernet/broadcom/tg3.c | 6 ------ include/linux/brcmphy.h | 9 +++------ 2 files changed, 3 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index 8936c2bc6286..d2381929931b 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -1580,12 +1580,6 @@ static int tg3_mdio_init(struct tg3 *tp) PHY_BRCM_RX_REFCLK_UNUSED | PHY_BRCM_DIS_TXCRXC_NOENRGY | PHY_BRCM_AUTO_PWRDWN_ENABLE; - if (tg3_flag(tp, RGMII_INBAND_DISABLE)) - phydev->dev_flags |= PHY_BRCM_STD_IBND_DISABLE; - if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN)) - phydev->dev_flags |= PHY_BRCM_EXT_IBND_RX_ENABLE; - if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN)) - phydev->dev_flags |= PHY_BRCM_EXT_IBND_TX_ENABLE; fallthrough; case PHY_ID_RTL8211C: phydev->interface = PHY_INTERFACE_MODE_RGMII; diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index 16597d3fa011..8fe1d55371ac 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -63,12 +63,9 @@ #define PHY_BRCM_AUTO_PWRDWN_ENABLE 0x00000001 #define PHY_BRCM_RX_REFCLK_UNUSED 0x00000002 -#define PHY_BRCM_STD_IBND_DISABLE 0x00000004 -#define PHY_BRCM_EXT_IBND_RX_ENABLE 0x00000008 -#define PHY_BRCM_EXT_IBND_TX_ENABLE 0x00000010 -#define PHY_BRCM_CLEAR_RGMII_MODE 0x00000020 -#define PHY_BRCM_DIS_TXCRXC_NOENRGY 0x00000040 -#define PHY_BRCM_EN_MASTER_MODE 0x00000080 +#define PHY_BRCM_CLEAR_RGMII_MODE 0x00000004 +#define PHY_BRCM_DIS_TXCRXC_NOENRGY 0x00000008 +#define PHY_BRCM_EN_MASTER_MODE 0x00000010 /* Broadcom BCM7xxx specific workarounds */ #define PHY_BRCM_7XXX_REV(x) (((x) >> 8) & 0xff) -- cgit v1.2.3 From 7331d1d4622ba7e668ec19cfba2ed7feb4a3084e Mon Sep 17 00:00:00 2001 From: Pavana Sharma Date: Tue, 16 Feb 2021 20:20:53 +0100 Subject: net: phy: Add 5GBASER interface mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 5GBASE-R phy interface mode Signed-off-by: Pavana Sharma Reviewed-by: Andrew Lunn Reviewed-by: Florian Fainelli Signed-off-by: Marek Behún Signed-off-by: David S. Miller --- Documentation/networking/phy.rst | 6 ++++++ include/linux/phy.h | 4 ++++ 2 files changed, 10 insertions(+) (limited to 'include/linux') diff --git a/Documentation/networking/phy.rst b/Documentation/networking/phy.rst index 70136cc9e25e..06adfc2afcf0 100644 --- a/Documentation/networking/phy.rst +++ b/Documentation/networking/phy.rst @@ -267,6 +267,12 @@ Some of the interface modes are described below: duplex, pause or other settings. This is dependent on the MAC and/or PHY behaviour. +``PHY_INTERFACE_MODE_5GBASER`` + This is the IEEE 802.3 Clause 129 defined 5GBASE-R protocol. It is + identical to the 10GBASE-R protocol defined in Clause 49, with the + exception that it operates at half the frequency. Please refer to the + IEEE standard for the definition. + ``PHY_INTERFACE_MODE_10GBASER`` This is the IEEE 802.3 Clause 49 defined 10GBASE-R protocol used with various different mediums. Please refer to the IEEE standard for a diff --git a/include/linux/phy.h b/include/linux/phy.h index 5d7c4084ade9..0d537f59b77f 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -107,6 +107,7 @@ extern const int phy_10gbit_features_array[1]; * @PHY_INTERFACE_MODE_100BASEX: 100 BaseX * @PHY_INTERFACE_MODE_1000BASEX: 1000 BaseX * @PHY_INTERFACE_MODE_2500BASEX: 2500 BaseX + * @PHY_INTERFACE_MODE_5GBASER: 5G BaseR * @PHY_INTERFACE_MODE_RXAUI: Reduced XAUI * @PHY_INTERFACE_MODE_XAUI: 10 Gigabit Attachment Unit Interface * @PHY_INTERFACE_MODE_10GBASER: 10G BaseR @@ -139,6 +140,7 @@ typedef enum { PHY_INTERFACE_MODE_100BASEX, PHY_INTERFACE_MODE_1000BASEX, PHY_INTERFACE_MODE_2500BASEX, + PHY_INTERFACE_MODE_5GBASER, PHY_INTERFACE_MODE_RXAUI, PHY_INTERFACE_MODE_XAUI, /* 10GBASE-R, XFI, SFI - single lane 10G Serdes */ @@ -209,6 +211,8 @@ static inline const char *phy_modes(phy_interface_t interface) return "1000base-x"; case PHY_INTERFACE_MODE_2500BASEX: return "2500base-x"; + case PHY_INTERFACE_MODE_5GBASER: + return "5gbase-r"; case PHY_INTERFACE_MODE_RXAUI: return "rxaui"; case PHY_INTERFACE_MODE_XAUI: -- cgit v1.2.3 From d8ea7ff3995ead5193313c72c0d97c9c16c83be9 Mon Sep 17 00:00:00 2001 From: Horatiu Vultur Date: Tue, 16 Feb 2021 22:42:03 +0100 Subject: net: mscc: ocelot: Add support for MRP Add basic support for MRP. The HW will just trap all MRP frames on the ring ports to CPU and allow the SW to process them. In this way it is possible to for this node to behave both as MRM and MRC. Current limitations are: - it doesn't support Interconnect roles. - it supports only a single ring. - the HW should be able to do forwarding of MRP Test frames so the SW will not need to do this. So it would be able to have the role MRC without SW support. Signed-off-by: Horatiu Vultur Signed-off-by: David S. Miller --- drivers/net/ethernet/mscc/Makefile | 1 + drivers/net/ethernet/mscc/ocelot.c | 10 +- drivers/net/ethernet/mscc/ocelot_mrp.c | 175 +++++++++++++++++++++++++++++++++ drivers/net/ethernet/mscc/ocelot_net.c | 60 +++++++++++ include/linux/dsa/ocelot.h | 5 + include/soc/mscc/ocelot.h | 45 +++++++++ 6 files changed, 295 insertions(+), 1 deletion(-) create mode 100644 drivers/net/ethernet/mscc/ocelot_mrp.c (limited to 'include/linux') diff --git a/drivers/net/ethernet/mscc/Makefile b/drivers/net/ethernet/mscc/Makefile index 346bba2730ad..722c27694b21 100644 --- a/drivers/net/ethernet/mscc/Makefile +++ b/drivers/net/ethernet/mscc/Makefile @@ -8,6 +8,7 @@ mscc_ocelot_switch_lib-y := \ ocelot_flower.o \ ocelot_ptp.o \ ocelot_devlink.o +mscc_ocelot_switch_lib-$(CONFIG_BRIDGE_MRP) += ocelot_mrp.o obj-$(CONFIG_MSCC_OCELOT_SWITCH) += mscc_ocelot.o mscc_ocelot-y := \ ocelot_vsc7514.o \ diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index 5d13087c85d6..46e5c9136bac 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -687,7 +687,7 @@ static int ocelot_xtr_poll_xfh(struct ocelot *ocelot, int grp, u32 *xfh) int ocelot_xtr_poll_frame(struct ocelot *ocelot, int grp, struct sk_buff **nskb) { struct skb_shared_hwtstamps *shhwtstamps; - u64 tod_in_ns, full_ts_in_ns; + u64 tod_in_ns, full_ts_in_ns, cpuq; u64 timestamp, src_port, len; u32 xfh[OCELOT_TAG_LEN / 4]; struct net_device *dev; @@ -704,6 +704,7 @@ int ocelot_xtr_poll_frame(struct ocelot *ocelot, int grp, struct sk_buff **nskb) ocelot_xfh_get_src_port(xfh, &src_port); ocelot_xfh_get_len(xfh, &len); ocelot_xfh_get_rew_val(xfh, ×tamp); + ocelot_xfh_get_cpuq(xfh, &cpuq); if (WARN_ON(src_port >= ocelot->num_phys_ports)) return -EINVAL; @@ -770,6 +771,13 @@ int ocelot_xtr_poll_frame(struct ocelot *ocelot, int grp, struct sk_buff **nskb) skb->offload_fwd_mark = 1; skb->protocol = eth_type_trans(skb, dev); + +#if IS_ENABLED(CONFIG_BRIDGE_MRP) + if (skb->protocol == cpu_to_be16(ETH_P_MRP) && + cpuq & BIT(OCELOT_MRP_CPUQ)) + skb->offload_fwd_mark = 0; +#endif + *nskb = skb; return 0; diff --git a/drivers/net/ethernet/mscc/ocelot_mrp.c b/drivers/net/ethernet/mscc/ocelot_mrp.c new file mode 100644 index 000000000000..683da320bfd8 --- /dev/null +++ b/drivers/net/ethernet/mscc/ocelot_mrp.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Microsemi Ocelot Switch driver + * + * This contains glue logic between the switchdev driver operations and the + * mscc_ocelot_switch_lib. + * + * Copyright (c) 2017, 2019 Microsemi Corporation + * Copyright 2020-2021 NXP Semiconductors + */ + +#include +#include +#include +#include +#include "ocelot.h" +#include "ocelot_vcap.h" + +static int ocelot_mrp_del_vcap(struct ocelot *ocelot, int port) +{ + struct ocelot_vcap_block *block_vcap_is2; + struct ocelot_vcap_filter *filter; + + block_vcap_is2 = &ocelot->block[VCAP_IS2]; + filter = ocelot_vcap_block_find_filter_by_id(block_vcap_is2, port, + false); + if (!filter) + return 0; + + return ocelot_vcap_filter_del(ocelot, filter); +} + +int ocelot_mrp_add(struct ocelot *ocelot, int port, + const struct switchdev_obj_mrp *mrp) +{ + struct ocelot_port *ocelot_port = ocelot->ports[port]; + struct ocelot_port_private *priv; + struct net_device *dev; + + if (!ocelot_port) + return -EOPNOTSUPP; + + priv = container_of(ocelot_port, struct ocelot_port_private, port); + dev = priv->dev; + + if (mrp->p_port != dev && mrp->s_port != dev) + return 0; + + if (ocelot->mrp_ring_id != 0 && + ocelot->mrp_s_port && + ocelot->mrp_p_port) + return -EINVAL; + + if (mrp->p_port == dev) + ocelot->mrp_p_port = dev; + + if (mrp->s_port == dev) + ocelot->mrp_s_port = dev; + + ocelot->mrp_ring_id = mrp->ring_id; + + return 0; +} +EXPORT_SYMBOL(ocelot_mrp_add); + +int ocelot_mrp_del(struct ocelot *ocelot, int port, + const struct switchdev_obj_mrp *mrp) +{ + struct ocelot_port *ocelot_port = ocelot->ports[port]; + struct ocelot_port_private *priv; + struct net_device *dev; + + if (!ocelot_port) + return -EOPNOTSUPP; + + priv = container_of(ocelot_port, struct ocelot_port_private, port); + dev = priv->dev; + + if (ocelot->mrp_p_port != dev && ocelot->mrp_s_port != dev) + return 0; + + if (ocelot->mrp_ring_id == 0 && + !ocelot->mrp_s_port && + !ocelot->mrp_p_port) + return -EINVAL; + + if (ocelot_mrp_del_vcap(ocelot, priv->chip_port)) + return -EINVAL; + + if (ocelot->mrp_p_port == dev) + ocelot->mrp_p_port = NULL; + + if (ocelot->mrp_s_port == dev) + ocelot->mrp_s_port = NULL; + + ocelot->mrp_ring_id = 0; + + return 0; +} +EXPORT_SYMBOL(ocelot_mrp_del); + +int ocelot_mrp_add_ring_role(struct ocelot *ocelot, int port, + const struct switchdev_obj_ring_role_mrp *mrp) +{ + struct ocelot_port *ocelot_port = ocelot->ports[port]; + struct ocelot_vcap_filter *filter; + struct ocelot_port_private *priv; + struct net_device *dev; + int err; + + if (!ocelot_port) + return -EOPNOTSUPP; + + priv = container_of(ocelot_port, struct ocelot_port_private, port); + dev = priv->dev; + + if (ocelot->mrp_ring_id != mrp->ring_id) + return -EINVAL; + + if (!mrp->sw_backup) + return -EOPNOTSUPP; + + if (ocelot->mrp_p_port != dev && ocelot->mrp_s_port != dev) + return 0; + + filter = kzalloc(sizeof(*filter), GFP_ATOMIC); + if (!filter) + return -ENOMEM; + + filter->key_type = OCELOT_VCAP_KEY_ETYPE; + filter->prio = 1; + filter->id.cookie = priv->chip_port; + filter->id.tc_offload = false; + filter->block_id = VCAP_IS2; + filter->type = OCELOT_VCAP_FILTER_OFFLOAD; + filter->ingress_port_mask = BIT(priv->chip_port); + *(__be16 *)filter->key.etype.etype.value = htons(ETH_P_MRP); + *(__be16 *)filter->key.etype.etype.mask = htons(0xffff); + filter->action.mask_mode = OCELOT_MASK_MODE_PERMIT_DENY; + filter->action.port_mask = 0x0; + filter->action.cpu_copy_ena = true; + filter->action.cpu_qu_num = OCELOT_MRP_CPUQ; + + err = ocelot_vcap_filter_add(ocelot, filter, NULL); + if (err) + kfree(filter); + + return err; +} +EXPORT_SYMBOL(ocelot_mrp_add_ring_role); + +int ocelot_mrp_del_ring_role(struct ocelot *ocelot, int port, + const struct switchdev_obj_ring_role_mrp *mrp) +{ + struct ocelot_port *ocelot_port = ocelot->ports[port]; + struct ocelot_port_private *priv; + struct net_device *dev; + + if (!ocelot_port) + return -EOPNOTSUPP; + + priv = container_of(ocelot_port, struct ocelot_port_private, port); + dev = priv->dev; + + if (ocelot->mrp_ring_id != mrp->ring_id) + return -EINVAL; + + if (!mrp->sw_backup) + return -EOPNOTSUPP; + + if (ocelot->mrp_p_port != dev && ocelot->mrp_s_port != dev) + return 0; + + return ocelot_mrp_del_vcap(ocelot, priv->chip_port); +} +EXPORT_SYMBOL(ocelot_mrp_del_ring_role); diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c index 6518262532f0..12cb6867a2d0 100644 --- a/drivers/net/ethernet/mscc/ocelot_net.c +++ b/drivers/net/ethernet/mscc/ocelot_net.c @@ -1010,6 +1010,52 @@ static int ocelot_port_obj_del_mdb(struct net_device *dev, return ocelot_port_mdb_del(ocelot, port, mdb); } +static int ocelot_port_obj_mrp_add(struct net_device *dev, + const struct switchdev_obj_mrp *mrp) +{ + struct ocelot_port_private *priv = netdev_priv(dev); + struct ocelot_port *ocelot_port = &priv->port; + struct ocelot *ocelot = ocelot_port->ocelot; + int port = priv->chip_port; + + return ocelot_mrp_add(ocelot, port, mrp); +} + +static int ocelot_port_obj_mrp_del(struct net_device *dev, + const struct switchdev_obj_mrp *mrp) +{ + struct ocelot_port_private *priv = netdev_priv(dev); + struct ocelot_port *ocelot_port = &priv->port; + struct ocelot *ocelot = ocelot_port->ocelot; + int port = priv->chip_port; + + return ocelot_mrp_del(ocelot, port, mrp); +} + +static int +ocelot_port_obj_mrp_add_ring_role(struct net_device *dev, + const struct switchdev_obj_ring_role_mrp *mrp) +{ + struct ocelot_port_private *priv = netdev_priv(dev); + struct ocelot_port *ocelot_port = &priv->port; + struct ocelot *ocelot = ocelot_port->ocelot; + int port = priv->chip_port; + + return ocelot_mrp_add_ring_role(ocelot, port, mrp); +} + +static int +ocelot_port_obj_mrp_del_ring_role(struct net_device *dev, + const struct switchdev_obj_ring_role_mrp *mrp) +{ + struct ocelot_port_private *priv = netdev_priv(dev); + struct ocelot_port *ocelot_port = &priv->port; + struct ocelot *ocelot = ocelot_port->ocelot; + int port = priv->chip_port; + + return ocelot_mrp_del_ring_role(ocelot, port, mrp); +} + static int ocelot_port_obj_add(struct net_device *dev, const struct switchdev_obj *obj, struct netlink_ext_ack *extack) @@ -1024,6 +1070,13 @@ static int ocelot_port_obj_add(struct net_device *dev, case SWITCHDEV_OBJ_ID_PORT_MDB: ret = ocelot_port_obj_add_mdb(dev, SWITCHDEV_OBJ_PORT_MDB(obj)); break; + case SWITCHDEV_OBJ_ID_MRP: + ret = ocelot_port_obj_mrp_add(dev, SWITCHDEV_OBJ_MRP(obj)); + break; + case SWITCHDEV_OBJ_ID_RING_ROLE_MRP: + ret = ocelot_port_obj_mrp_add_ring_role(dev, + SWITCHDEV_OBJ_RING_ROLE_MRP(obj)); + break; default: return -EOPNOTSUPP; } @@ -1044,6 +1097,13 @@ static int ocelot_port_obj_del(struct net_device *dev, case SWITCHDEV_OBJ_ID_PORT_MDB: ret = ocelot_port_obj_del_mdb(dev, SWITCHDEV_OBJ_PORT_MDB(obj)); break; + case SWITCHDEV_OBJ_ID_MRP: + ret = ocelot_port_obj_mrp_del(dev, SWITCHDEV_OBJ_MRP(obj)); + break; + case SWITCHDEV_OBJ_ID_RING_ROLE_MRP: + ret = ocelot_port_obj_mrp_del_ring_role(dev, + SWITCHDEV_OBJ_RING_ROLE_MRP(obj)); + break; default: return -EOPNOTSUPP; } diff --git a/include/linux/dsa/ocelot.h b/include/linux/dsa/ocelot.h index c6bc45ae5e03..4265f328681a 100644 --- a/include/linux/dsa/ocelot.h +++ b/include/linux/dsa/ocelot.h @@ -160,6 +160,11 @@ static inline void ocelot_xfh_get_src_port(void *extraction, u64 *src_port) packing(extraction, src_port, 46, 43, OCELOT_TAG_LEN, UNPACK, 0); } +static inline void ocelot_xfh_get_cpuq(void *extraction, u64 *cpuq) +{ + packing(extraction, cpuq, 28, 20, OCELOT_TAG_LEN, UNPACK, 0); +} + static inline void ocelot_xfh_get_qos_class(void *extraction, u64 *qos_class) { packing(extraction, qos_class, 19, 17, OCELOT_TAG_LEN, UNPACK, 0); diff --git a/include/soc/mscc/ocelot.h b/include/soc/mscc/ocelot.h index 1f2d90976564..425ff29d9389 100644 --- a/include/soc/mscc/ocelot.h +++ b/include/soc/mscc/ocelot.h @@ -112,6 +112,8 @@ #define REG_RESERVED_ADDR 0xffffffff #define REG_RESERVED(reg) REG(reg, REG_RESERVED_ADDR) +#define OCELOT_MRP_CPUQ 7 + enum ocelot_target { ANA = 1, QS, @@ -677,6 +679,12 @@ struct ocelot { /* Protects the PTP clock */ spinlock_t ptp_clock_lock; struct ptp_pin_desc ptp_pins[OCELOT_PTP_PINS_NUM]; + +#if IS_ENABLED(CONFIG_BRIDGE_MRP) + u16 mrp_ring_id; + struct net_device *mrp_p_port; + struct net_device *mrp_s_port; +#endif }; struct ocelot_policer { @@ -874,4 +882,41 @@ int ocelot_sb_occ_tc_port_bind_get(struct ocelot *ocelot, int port, enum devlink_sb_pool_type pool_type, u32 *p_cur, u32 *p_max); +#if IS_ENABLED(CONFIG_BRIDGE_MRP) +int ocelot_mrp_add(struct ocelot *ocelot, int port, + const struct switchdev_obj_mrp *mrp); +int ocelot_mrp_del(struct ocelot *ocelot, int port, + const struct switchdev_obj_mrp *mrp); +int ocelot_mrp_add_ring_role(struct ocelot *ocelot, int port, + const struct switchdev_obj_ring_role_mrp *mrp); +int ocelot_mrp_del_ring_role(struct ocelot *ocelot, int port, + const struct switchdev_obj_ring_role_mrp *mrp); +#else +static inline int ocelot_mrp_add(struct ocelot *ocelot, int port, + const struct switchdev_obj_mrp *mrp) +{ + return -EOPNOTSUPP; +} + +static inline int ocelot_mrp_del(struct ocelot *ocelot, int port, + const struct switchdev_obj_mrp *mrp) +{ + return -EOPNOTSUPP; +} + +static inline int +ocelot_mrp_add_ring_role(struct ocelot *ocelot, int port, + const struct switchdev_obj_ring_role_mrp *mrp) +{ + return -EOPNOTSUPP; +} + +static inline int +ocelot_mrp_del_ring_role(struct ocelot *ocelot, int port, + const struct switchdev_obj_ring_role_mrp *mrp) +{ + return -EOPNOTSUPP; +} +#endif + #endif -- cgit v1.2.3 From 3afd0218992a8d1398e9791d6c2edd4c948ae7ee Mon Sep 17 00:00:00 2001 From: Robert Hancock Date: Tue, 16 Feb 2021 16:54:52 -0600 Subject: net: phy: broadcom: Set proper 1000BaseX/SGMII interface mode for BCM54616S The default configuration for the BCM54616S PHY may not match the desired mode when using 1000BaseX or SGMII interface modes, such as when it is on an SFP module. Add code to explicitly set the correct mode using programming sequences provided by Bel-Fuse: https://www.belfuse.com/resources/datasheets/powersolutions/ds-bps-sfp-1gbt-05-series.pdf https://www.belfuse.com/resources/datasheets/powersolutions/ds-bps-sfp-1gbt-06-series.pdf Signed-off-by: Robert Hancock Acked-by: Florian Fainelli Signed-off-by: David S. Miller --- drivers/net/phy/broadcom.c | 84 +++++++++++++++++++++++++++++++++++++++------- include/linux/brcmphy.h | 4 +++ 2 files changed, 76 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c index 91fbd26c809e..8e7fc3368380 100644 --- a/drivers/net/phy/broadcom.c +++ b/drivers/net/phy/broadcom.c @@ -103,6 +103,64 @@ static int bcm54612e_config_init(struct phy_device *phydev) return 0; } +static int bcm54616s_config_init(struct phy_device *phydev) +{ + int rc, val; + + if (phydev->interface != PHY_INTERFACE_MODE_SGMII && + phydev->interface != PHY_INTERFACE_MODE_1000BASEX) + return 0; + + /* Ensure proper interface mode is selected. */ + /* Disable RGMII mode */ + val = bcm54xx_auxctl_read(phydev, MII_BCM54XX_AUXCTL_SHDWSEL_MISC); + if (val < 0) + return val; + val &= ~MII_BCM54XX_AUXCTL_SHDWSEL_MISC_RGMII_EN; + val |= MII_BCM54XX_AUXCTL_MISC_WREN; + rc = bcm54xx_auxctl_write(phydev, MII_BCM54XX_AUXCTL_SHDWSEL_MISC, + val); + if (rc < 0) + return rc; + + /* Select 1000BASE-X register set (primary SerDes) */ + val = bcm_phy_read_shadow(phydev, BCM54XX_SHD_MODE); + if (val < 0) + return val; + val |= BCM54XX_SHD_MODE_1000BX; + rc = bcm_phy_write_shadow(phydev, BCM54XX_SHD_MODE, val); + if (rc < 0) + return rc; + + /* Power down SerDes interface */ + rc = phy_set_bits(phydev, MII_BMCR, BMCR_PDOWN); + if (rc < 0) + return rc; + + /* Select proper interface mode */ + val &= ~BCM54XX_SHD_INTF_SEL_MASK; + val |= phydev->interface == PHY_INTERFACE_MODE_SGMII ? + BCM54XX_SHD_INTF_SEL_SGMII : + BCM54XX_SHD_INTF_SEL_GBIC; + rc = bcm_phy_write_shadow(phydev, BCM54XX_SHD_MODE, val); + if (rc < 0) + return rc; + + /* Power up SerDes interface */ + rc = phy_clear_bits(phydev, MII_BMCR, BMCR_PDOWN); + if (rc < 0) + return rc; + + /* Select copper register set */ + val &= ~BCM54XX_SHD_MODE_1000BX; + rc = bcm_phy_write_shadow(phydev, BCM54XX_SHD_MODE, val); + if (rc < 0) + return rc; + + /* Power up copper interface */ + return phy_clear_bits(phydev, MII_BMCR, BMCR_PDOWN); +} + /* Needs SMDSP clock enabled via bcm54xx_phydsp_config() */ static int bcm50610_a0_workaround(struct phy_device *phydev) { @@ -283,15 +341,17 @@ static int bcm54xx_config_init(struct phy_device *phydev) bcm54xx_adjust_rxrefclk(phydev); - if (BRCM_PHY_MODEL(phydev) == PHY_ID_BCM54210E) { + switch (BRCM_PHY_MODEL(phydev)) { + case PHY_ID_BCM54210E: err = bcm54210e_config_init(phydev); - if (err) - return err; - } else if (BRCM_PHY_MODEL(phydev) == PHY_ID_BCM54612E) { + break; + case PHY_ID_BCM54612E: err = bcm54612e_config_init(phydev); - if (err) - return err; - } else if (BRCM_PHY_MODEL(phydev) == PHY_ID_BCM54810) { + break; + case PHY_ID_BCM54616S: + err = bcm54616s_config_init(phydev); + break; + case PHY_ID_BCM54810: /* For BCM54810, we need to disable BroadR-Reach function */ val = bcm_phy_read_exp(phydev, BCM54810_EXP_BROADREACH_LRE_MISC_CTL); @@ -299,9 +359,10 @@ static int bcm54xx_config_init(struct phy_device *phydev) err = bcm_phy_write_exp(phydev, BCM54810_EXP_BROADREACH_LRE_MISC_CTL, val); - if (err < 0) - return err; + break; } + if (err) + return err; bcm54xx_phydsp_config(phydev); @@ -390,7 +451,7 @@ struct bcm54616s_phy_priv { static int bcm54616s_probe(struct phy_device *phydev) { struct bcm54616s_phy_priv *priv; - int val, intf_sel; + int val; priv = devm_kzalloc(&phydev->mdio.dev, sizeof(*priv), GFP_KERNEL); if (!priv) @@ -408,8 +469,7 @@ static int bcm54616s_probe(struct phy_device *phydev) * RGMII-1000Base-X is properly supported, but RGMII-100Base-FX * support is still missing as of now. */ - intf_sel = (val & BCM54XX_SHD_INTF_SEL_MASK) >> 1; - if (intf_sel == 1) { + if ((val & BCM54XX_SHD_INTF_SEL_MASK) == BCM54XX_SHD_INTF_SEL_RGMII) { val = bcm_phy_read_shadow(phydev, BCM54616S_SHD_100FX_CTRL); if (val < 0) return val; diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index 8fe1d55371ac..c2c2147dfeb8 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -129,6 +129,7 @@ #define MII_BCM54XX_AUXCTL_SHDWSEL_MISC 0x07 #define MII_BCM54XX_AUXCTL_SHDWSEL_MISC_WIRESPEED_EN 0x0010 +#define MII_BCM54XX_AUXCTL_SHDWSEL_MISC_RGMII_EN 0x0080 #define MII_BCM54XX_AUXCTL_SHDWSEL_MISC_RGMII_SKEW_EN 0x0100 #define MII_BCM54XX_AUXCTL_MISC_FORCE_AMDIX 0x0200 #define MII_BCM54XX_AUXCTL_MISC_WREN 0x8000 @@ -216,6 +217,9 @@ /* 11111: Mode Control Register */ #define BCM54XX_SHD_MODE 0x1f #define BCM54XX_SHD_INTF_SEL_MASK GENMASK(2, 1) /* INTERF_SEL[1:0] */ +#define BCM54XX_SHD_INTF_SEL_RGMII 0x02 +#define BCM54XX_SHD_INTF_SEL_SGMII 0x04 +#define BCM54XX_SHD_INTF_SEL_GBIC 0x06 #define BCM54XX_SHD_MODE_1000BX BIT(0) /* Enable 1000-X registers */ /* -- cgit v1.2.3 From b834489bceccc64641684eee5e93275cdf5f465b Mon Sep 17 00:00:00 2001 From: Robert Hancock Date: Tue, 16 Feb 2021 16:54:53 -0600 Subject: net: phy: Add is_on_sfp_module flag and phy_on_sfp helper Add a flag and helper function to indicate that a PHY device is part of an SFP module, which is set on attach. This can be used by PHY drivers to handle SFP-specific quirks or behavior. Signed-off-by: Robert Hancock Signed-off-by: David S. Miller --- drivers/net/phy/phy_device.c | 2 ++ include/linux/phy.h | 11 +++++++++++ 2 files changed, 13 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 05261698bf74..d6ac3ed38197 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -1377,6 +1377,8 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev, if (phydev->sfp_bus_attached) dev->sfp_bus = phydev->sfp_bus; + else if (dev->sfp_bus) + phydev->is_on_sfp_module = true; } /* Some Ethernet drivers try to connect to a PHY device before diff --git a/include/linux/phy.h b/include/linux/phy.h index 0d537f59b77f..1a12e4436b5b 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -492,6 +492,7 @@ struct macsec_ops; * @sysfs_links: Internal boolean tracking sysfs symbolic links setup/removal. * @loopback_enabled: Set true if this PHY has been loopbacked successfully. * @downshifted_rate: Set true if link speed has been downshifted. + * @is_on_sfp_module: Set true if PHY is located on an SFP module. * @state: State of the PHY for management purposes * @dev_flags: Device-specific flags used by the PHY driver. * @irq: IRQ number of the PHY's interrupt (-1 if none) @@ -565,6 +566,7 @@ struct phy_device { unsigned sysfs_links:1; unsigned loopback_enabled:1; unsigned downshifted_rate:1; + unsigned is_on_sfp_module:1; unsigned autoneg:1; /* The most recently read link state */ @@ -1296,6 +1298,15 @@ static inline bool phy_is_internal(struct phy_device *phydev) return phydev->is_internal; } +/** + * phy_on_sfp - Convenience function for testing if a PHY is on an SFP module + * @phydev: the phy_device struct + */ +static inline bool phy_on_sfp(struct phy_device *phydev) +{ + return phydev->is_on_sfp_module; +} + /** * phy_interface_mode_is_rgmii - Convenience function for testing if a * PHY interface mode is RGMII (all variants) -- cgit v1.2.3 From 2d24dd5798d0474d9bf705bfca8725e7d20f9d54 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 29 Apr 2020 17:03:22 +0200 Subject: rbtree: Add generic add and find helpers I've always been bothered by the endless (fragile) boilerplate for rbtree, and I recently wrote some rbtree helpers for objtool and figured I should lift them into the kernel and use them more widely. Provide: partial-order; less() based: - rb_add(): add a new entry to the rbtree - rb_add_cached(): like rb_add(), but for a rb_root_cached total-order; cmp() based: - rb_find(): find an entry in an rbtree - rb_find_add(): find an entry, and add if not found - rb_find_first(): find the first (leftmost) matching entry - rb_next_match(): continue from rb_find_first() - rb_for_each(): iterate a sub-tree using the previous two Inlining and constant propagation should see the compiler inline the whole thing, including the various compare functions. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Michel Lespinasse Acked-by: Davidlohr Bueso --- include/linux/rbtree.h | 190 ++++++++++++++++++++++++++++++++++++++++++ tools/include/linux/rbtree.h | 192 ++++++++++++++++++++++++++++++++++++++++++- tools/objtool/elf.c | 73 +++------------- 3 files changed, 392 insertions(+), 63 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rbtree.h b/include/linux/rbtree.h index d7db17996322..e0b300de8f3f 100644 --- a/include/linux/rbtree.h +++ b/include/linux/rbtree.h @@ -158,4 +158,194 @@ static inline void rb_replace_node_cached(struct rb_node *victim, rb_replace_node(victim, new, &root->rb_root); } +/* + * The below helper functions use 2 operators with 3 different + * calling conventions. The operators are related like: + * + * comp(a->key,b) < 0 := less(a,b) + * comp(a->key,b) > 0 := less(b,a) + * comp(a->key,b) == 0 := !less(a,b) && !less(b,a) + * + * If these operators define a partial order on the elements we make no + * guarantee on which of the elements matching the key is found. See + * rb_find(). + * + * The reason for this is to allow the find() interface without requiring an + * on-stack dummy object, which might not be feasible due to object size. + */ + +/** + * rb_add_cached() - insert @node into the leftmost cached tree @tree + * @node: node to insert + * @tree: leftmost cached tree to insert @node into + * @less: operator defining the (partial) node order + */ +static __always_inline void +rb_add_cached(struct rb_node *node, struct rb_root_cached *tree, + bool (*less)(struct rb_node *, const struct rb_node *)) +{ + struct rb_node **link = &tree->rb_root.rb_node; + struct rb_node *parent = NULL; + bool leftmost = true; + + while (*link) { + parent = *link; + if (less(node, parent)) { + link = &parent->rb_left; + } else { + link = &parent->rb_right; + leftmost = false; + } + } + + rb_link_node(node, parent, link); + rb_insert_color_cached(node, tree, leftmost); +} + +/** + * rb_add() - insert @node into @tree + * @node: node to insert + * @tree: tree to insert @node into + * @less: operator defining the (partial) node order + */ +static __always_inline void +rb_add(struct rb_node *node, struct rb_root *tree, + bool (*less)(struct rb_node *, const struct rb_node *)) +{ + struct rb_node **link = &tree->rb_node; + struct rb_node *parent = NULL; + + while (*link) { + parent = *link; + if (less(node, parent)) + link = &parent->rb_left; + else + link = &parent->rb_right; + } + + rb_link_node(node, parent, link); + rb_insert_color(node, tree); +} + +/** + * rb_find_add() - find equivalent @node in @tree, or add @node + * @node: node to look-for / insert + * @tree: tree to search / modify + * @cmp: operator defining the node order + * + * Returns the rb_node matching @node, or NULL when no match is found and @node + * is inserted. + */ +static __always_inline struct rb_node * +rb_find_add(struct rb_node *node, struct rb_root *tree, + int (*cmp)(struct rb_node *, const struct rb_node *)) +{ + struct rb_node **link = &tree->rb_node; + struct rb_node *parent = NULL; + int c; + + while (*link) { + parent = *link; + c = cmp(node, parent); + + if (c < 0) + link = &parent->rb_left; + else if (c > 0) + link = &parent->rb_right; + else + return parent; + } + + rb_link_node(node, parent, link); + rb_insert_color(node, tree); + return NULL; +} + +/** + * rb_find() - find @key in tree @tree + * @key: key to match + * @tree: tree to search + * @cmp: operator defining the node order + * + * Returns the rb_node matching @key or NULL. + */ +static __always_inline struct rb_node * +rb_find(const void *key, const struct rb_root *tree, + int (*cmp)(const void *key, const struct rb_node *)) +{ + struct rb_node *node = tree->rb_node; + + while (node) { + int c = cmp(key, node); + + if (c < 0) + node = node->rb_left; + else if (c > 0) + node = node->rb_right; + else + return node; + } + + return NULL; +} + +/** + * rb_find_first() - find the first @key in @tree + * @key: key to match + * @tree: tree to search + * @cmp: operator defining node order + * + * Returns the leftmost node matching @key, or NULL. + */ +static __always_inline struct rb_node * +rb_find_first(const void *key, const struct rb_root *tree, + int (*cmp)(const void *key, const struct rb_node *)) +{ + struct rb_node *node = tree->rb_node; + struct rb_node *match = NULL; + + while (node) { + int c = cmp(key, node); + + if (c <= 0) { + if (!c) + match = node; + node = node->rb_left; + } else if (c > 0) { + node = node->rb_right; + } + } + + return match; +} + +/** + * rb_next_match() - find the next @key in @tree + * @key: key to match + * @tree: tree to search + * @cmp: operator defining node order + * + * Returns the next node matching @key, or NULL. + */ +static __always_inline struct rb_node * +rb_next_match(const void *key, struct rb_node *node, + int (*cmp)(const void *key, const struct rb_node *)) +{ + node = rb_next(node); + if (node && cmp(key, node)) + node = NULL; + return node; +} + +/** + * rb_for_each() - iterates a subtree matching @key + * @node: iterator + * @key: key to match + * @tree: tree to search + * @cmp: operator defining node order + */ +#define rb_for_each(node, key, tree, cmp) \ + for ((node) = rb_find_first((key), (tree), (cmp)); \ + (node); (node) = rb_next_match((key), (node), (cmp))) + #endif /* _LINUX_RBTREE_H */ diff --git a/tools/include/linux/rbtree.h b/tools/include/linux/rbtree.h index 30dd21f976c3..2680f2edb837 100644 --- a/tools/include/linux/rbtree.h +++ b/tools/include/linux/rbtree.h @@ -152,4 +152,194 @@ static inline void rb_replace_node_cached(struct rb_node *victim, rb_replace_node(victim, new, &root->rb_root); } -#endif /* __TOOLS_LINUX_PERF_RBTREE_H */ +/* + * The below helper functions use 2 operators with 3 different + * calling conventions. The operators are related like: + * + * comp(a->key,b) < 0 := less(a,b) + * comp(a->key,b) > 0 := less(b,a) + * comp(a->key,b) == 0 := !less(a,b) && !less(b,a) + * + * If these operators define a partial order on the elements we make no + * guarantee on which of the elements matching the key is found. See + * rb_find(). + * + * The reason for this is to allow the find() interface without requiring an + * on-stack dummy object, which might not be feasible due to object size. + */ + +/** + * rb_add_cached() - insert @node into the leftmost cached tree @tree + * @node: node to insert + * @tree: leftmost cached tree to insert @node into + * @less: operator defining the (partial) node order + */ +static __always_inline void +rb_add_cached(struct rb_node *node, struct rb_root_cached *tree, + bool (*less)(struct rb_node *, const struct rb_node *)) +{ + struct rb_node **link = &tree->rb_root.rb_node; + struct rb_node *parent = NULL; + bool leftmost = true; + + while (*link) { + parent = *link; + if (less(node, parent)) { + link = &parent->rb_left; + } else { + link = &parent->rb_right; + leftmost = false; + } + } + + rb_link_node(node, parent, link); + rb_insert_color_cached(node, tree, leftmost); +} + +/** + * rb_add() - insert @node into @tree + * @node: node to insert + * @tree: tree to insert @node into + * @less: operator defining the (partial) node order + */ +static __always_inline void +rb_add(struct rb_node *node, struct rb_root *tree, + bool (*less)(struct rb_node *, const struct rb_node *)) +{ + struct rb_node **link = &tree->rb_node; + struct rb_node *parent = NULL; + + while (*link) { + parent = *link; + if (less(node, parent)) + link = &parent->rb_left; + else + link = &parent->rb_right; + } + + rb_link_node(node, parent, link); + rb_insert_color(node, tree); +} + +/** + * rb_find_add() - find equivalent @node in @tree, or add @node + * @node: node to look-for / insert + * @tree: tree to search / modify + * @cmp: operator defining the node order + * + * Returns the rb_node matching @node, or NULL when no match is found and @node + * is inserted. + */ +static __always_inline struct rb_node * +rb_find_add(struct rb_node *node, struct rb_root *tree, + int (*cmp)(struct rb_node *, const struct rb_node *)) +{ + struct rb_node **link = &tree->rb_node; + struct rb_node *parent = NULL; + int c; + + while (*link) { + parent = *link; + c = cmp(node, parent); + + if (c < 0) + link = &parent->rb_left; + else if (c > 0) + link = &parent->rb_right; + else + return parent; + } + + rb_link_node(node, parent, link); + rb_insert_color(node, tree); + return NULL; +} + +/** + * rb_find() - find @key in tree @tree + * @key: key to match + * @tree: tree to search + * @cmp: operator defining the node order + * + * Returns the rb_node matching @key or NULL. + */ +static __always_inline struct rb_node * +rb_find(const void *key, const struct rb_root *tree, + int (*cmp)(const void *key, const struct rb_node *)) +{ + struct rb_node *node = tree->rb_node; + + while (node) { + int c = cmp(key, node); + + if (c < 0) + node = node->rb_left; + else if (c > 0) + node = node->rb_right; + else + return node; + } + + return NULL; +} + +/** + * rb_find_first() - find the first @key in @tree + * @key: key to match + * @tree: tree to search + * @cmp: operator defining node order + * + * Returns the leftmost node matching @key, or NULL. + */ +static __always_inline struct rb_node * +rb_find_first(const void *key, const struct rb_root *tree, + int (*cmp)(const void *key, const struct rb_node *)) +{ + struct rb_node *node = tree->rb_node; + struct rb_node *match = NULL; + + while (node) { + int c = cmp(key, node); + + if (c <= 0) { + if (!c) + match = node; + node = node->rb_left; + } else if (c > 0) { + node = node->rb_right; + } + } + + return match; +} + +/** + * rb_next_match() - find the next @key in @tree + * @key: key to match + * @tree: tree to search + * @cmp: operator defining node order + * + * Returns the next node matching @key, or NULL. + */ +static __always_inline struct rb_node * +rb_next_match(const void *key, struct rb_node *node, + int (*cmp)(const void *key, const struct rb_node *)) +{ + node = rb_next(node); + if (node && cmp(key, node)) + node = NULL; + return node; +} + +/** + * rb_for_each() - iterates a subtree matching @key + * @node: iterator + * @key: key to match + * @tree: tree to search + * @cmp: operator defining node order + */ +#define rb_for_each(node, key, tree, cmp) \ + for ((node) = rb_find_first((key), (tree), (cmp)); \ + (node); (node) = rb_next_match((key), (node), (cmp))) + +#endif /* __TOOLS_LINUX_PERF_RBTREE_H */ diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index d8421e1d06be..e85988ce04f1 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -43,75 +43,24 @@ static void elf_hash_init(struct hlist_head *table) #define elf_hash_for_each_possible(name, obj, member, key) \ hlist_for_each_entry(obj, &name[hash_min(key, elf_hash_bits())], member) -static void rb_add(struct rb_root *tree, struct rb_node *node, - int (*cmp)(struct rb_node *, const struct rb_node *)) -{ - struct rb_node **link = &tree->rb_node; - struct rb_node *parent = NULL; - - while (*link) { - parent = *link; - if (cmp(node, parent) < 0) - link = &parent->rb_left; - else - link = &parent->rb_right; - } - - rb_link_node(node, parent, link); - rb_insert_color(node, tree); -} - -static struct rb_node *rb_find_first(const struct rb_root *tree, const void *key, - int (*cmp)(const void *key, const struct rb_node *)) -{ - struct rb_node *node = tree->rb_node; - struct rb_node *match = NULL; - - while (node) { - int c = cmp(key, node); - if (c <= 0) { - if (!c) - match = node; - node = node->rb_left; - } else if (c > 0) { - node = node->rb_right; - } - } - - return match; -} - -static struct rb_node *rb_next_match(struct rb_node *node, const void *key, - int (*cmp)(const void *key, const struct rb_node *)) -{ - node = rb_next(node); - if (node && cmp(key, node)) - node = NULL; - return node; -} - -#define rb_for_each(tree, node, key, cmp) \ - for ((node) = rb_find_first((tree), (key), (cmp)); \ - (node); (node) = rb_next_match((node), (key), (cmp))) - -static int symbol_to_offset(struct rb_node *a, const struct rb_node *b) +static bool symbol_to_offset(struct rb_node *a, const struct rb_node *b) { struct symbol *sa = rb_entry(a, struct symbol, node); struct symbol *sb = rb_entry(b, struct symbol, node); if (sa->offset < sb->offset) - return -1; + return true; if (sa->offset > sb->offset) - return 1; + return false; if (sa->len < sb->len) - return -1; + return true; if (sa->len > sb->len) - return 1; + return false; sa->alias = sb; - return 0; + return false; } static int symbol_by_offset(const void *key, const struct rb_node *node) @@ -165,7 +114,7 @@ struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset) { struct rb_node *node; - rb_for_each(&sec->symbol_tree, node, &offset, symbol_by_offset) { + rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) { struct symbol *s = rb_entry(node, struct symbol, node); if (s->offset == offset && s->type != STT_SECTION) @@ -179,7 +128,7 @@ struct symbol *find_func_by_offset(struct section *sec, unsigned long offset) { struct rb_node *node; - rb_for_each(&sec->symbol_tree, node, &offset, symbol_by_offset) { + rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) { struct symbol *s = rb_entry(node, struct symbol, node); if (s->offset == offset && s->type == STT_FUNC) @@ -193,7 +142,7 @@ struct symbol *find_symbol_containing(const struct section *sec, unsigned long o { struct rb_node *node; - rb_for_each(&sec->symbol_tree, node, &offset, symbol_by_offset) { + rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) { struct symbol *s = rb_entry(node, struct symbol, node); if (s->type != STT_SECTION) @@ -207,7 +156,7 @@ struct symbol *find_func_containing(struct section *sec, unsigned long offset) { struct rb_node *node; - rb_for_each(&sec->symbol_tree, node, &offset, symbol_by_offset) { + rb_for_each(node, &offset, &sec->symbol_tree, symbol_by_offset) { struct symbol *s = rb_entry(node, struct symbol, node); if (s->type == STT_FUNC) @@ -442,7 +391,7 @@ static int read_symbols(struct elf *elf) sym->offset = sym->sym.st_value; sym->len = sym->sym.st_size; - rb_add(&sym->sec->symbol_tree, &sym->node, symbol_to_offset); + rb_add(&sym->node, &sym->sec->symbol_tree, symbol_to_offset); pnode = rb_prev(&sym->node); if (pnode) entry = &rb_entry(pnode, struct symbol, node)->list; -- cgit v1.2.3 From 8ecca39483ed4e4e97096d0d6f8e25fdd323b189 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 29 Apr 2020 17:04:41 +0200 Subject: rbtree, sched/deadline: Use rb_add_cached() Reduce rbtree boiler plate by using the new helpers. Make rb_add_cached() / rb_erase_cached() return a pointer to the leftmost node to aid in updating additional state. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Acked-by: Davidlohr Bueso --- include/linux/rbtree.h | 18 +++++++++--- kernel/sched/deadline.c | 77 ++++++++++++++++++------------------------------- 2 files changed, 42 insertions(+), 53 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rbtree.h b/include/linux/rbtree.h index e0b300de8f3f..d31ecaf4fdd3 100644 --- a/include/linux/rbtree.h +++ b/include/linux/rbtree.h @@ -141,12 +141,18 @@ static inline void rb_insert_color_cached(struct rb_node *node, rb_insert_color(node, &root->rb_root); } -static inline void rb_erase_cached(struct rb_node *node, - struct rb_root_cached *root) + +static inline struct rb_node * +rb_erase_cached(struct rb_node *node, struct rb_root_cached *root) { + struct rb_node *leftmost = NULL; + if (root->rb_leftmost == node) - root->rb_leftmost = rb_next(node); + leftmost = root->rb_leftmost = rb_next(node); + rb_erase(node, &root->rb_root); + + return leftmost; } static inline void rb_replace_node_cached(struct rb_node *victim, @@ -179,8 +185,10 @@ static inline void rb_replace_node_cached(struct rb_node *victim, * @node: node to insert * @tree: leftmost cached tree to insert @node into * @less: operator defining the (partial) node order + * + * Returns @node when it is the new leftmost, or NULL. */ -static __always_inline void +static __always_inline struct rb_node * rb_add_cached(struct rb_node *node, struct rb_root_cached *tree, bool (*less)(struct rb_node *, const struct rb_node *)) { @@ -200,6 +208,8 @@ rb_add_cached(struct rb_node *node, struct rb_root_cached *tree, rb_link_node(node, parent, link); rb_insert_color_cached(node, tree, leftmost); + + return leftmost ? node : NULL; } /** diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 5421782fe897..1508d126e88b 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -517,58 +517,44 @@ static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) update_dl_migration(dl_rq); } +#define __node_2_pdl(node) \ + rb_entry((node), struct task_struct, pushable_dl_tasks) + +static inline bool __pushable_less(struct rb_node *a, const struct rb_node *b) +{ + return dl_entity_preempt(&__node_2_pdl(a)->dl, &__node_2_pdl(b)->dl); +} + /* * The list of pushable -deadline task is not a plist, like in * sched_rt.c, it is an rb-tree with tasks ordered by deadline. */ static void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p) { - struct dl_rq *dl_rq = &rq->dl; - struct rb_node **link = &dl_rq->pushable_dl_tasks_root.rb_root.rb_node; - struct rb_node *parent = NULL; - struct task_struct *entry; - bool leftmost = true; + struct rb_node *leftmost; BUG_ON(!RB_EMPTY_NODE(&p->pushable_dl_tasks)); - while (*link) { - parent = *link; - entry = rb_entry(parent, struct task_struct, - pushable_dl_tasks); - if (dl_entity_preempt(&p->dl, &entry->dl)) - link = &parent->rb_left; - else { - link = &parent->rb_right; - leftmost = false; - } - } - + leftmost = rb_add_cached(&p->pushable_dl_tasks, + &rq->dl.pushable_dl_tasks_root, + __pushable_less); if (leftmost) - dl_rq->earliest_dl.next = p->dl.deadline; - - rb_link_node(&p->pushable_dl_tasks, parent, link); - rb_insert_color_cached(&p->pushable_dl_tasks, - &dl_rq->pushable_dl_tasks_root, leftmost); + rq->dl.earliest_dl.next = p->dl.deadline; } static void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p) { struct dl_rq *dl_rq = &rq->dl; + struct rb_root_cached *root = &dl_rq->pushable_dl_tasks_root; + struct rb_node *leftmost; if (RB_EMPTY_NODE(&p->pushable_dl_tasks)) return; - if (dl_rq->pushable_dl_tasks_root.rb_leftmost == &p->pushable_dl_tasks) { - struct rb_node *next_node; - - next_node = rb_next(&p->pushable_dl_tasks); - if (next_node) { - dl_rq->earliest_dl.next = rb_entry(next_node, - struct task_struct, pushable_dl_tasks)->dl.deadline; - } - } + leftmost = rb_erase_cached(&p->pushable_dl_tasks, root); + if (leftmost) + dl_rq->earliest_dl.next = __node_2_pdl(leftmost)->dl.deadline; - rb_erase_cached(&p->pushable_dl_tasks, &dl_rq->pushable_dl_tasks_root); RB_CLEAR_NODE(&p->pushable_dl_tasks); } @@ -1478,29 +1464,21 @@ void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) dec_dl_migration(dl_se, dl_rq); } +#define __node_2_dle(node) \ + rb_entry((node), struct sched_dl_entity, rb_node) + +static inline bool __dl_less(struct rb_node *a, const struct rb_node *b) +{ + return dl_time_before(__node_2_dle(a)->deadline, __node_2_dle(b)->deadline); +} + static void __enqueue_dl_entity(struct sched_dl_entity *dl_se) { struct dl_rq *dl_rq = dl_rq_of_se(dl_se); - struct rb_node **link = &dl_rq->root.rb_root.rb_node; - struct rb_node *parent = NULL; - struct sched_dl_entity *entry; - int leftmost = 1; BUG_ON(!RB_EMPTY_NODE(&dl_se->rb_node)); - while (*link) { - parent = *link; - entry = rb_entry(parent, struct sched_dl_entity, rb_node); - if (dl_time_before(dl_se->deadline, entry->deadline)) - link = &parent->rb_left; - else { - link = &parent->rb_right; - leftmost = 0; - } - } - - rb_link_node(&dl_se->rb_node, parent, link); - rb_insert_color_cached(&dl_se->rb_node, &dl_rq->root, leftmost); + rb_add_cached(&dl_se->rb_node, &dl_rq->root, __dl_less); inc_dl_tasks(dl_se, dl_rq); } @@ -1513,6 +1491,7 @@ static void __dequeue_dl_entity(struct sched_dl_entity *dl_se) return; rb_erase_cached(&dl_se->rb_node, &dl_rq->root); + RB_CLEAR_NODE(&dl_se->rb_node); dec_dl_tasks(dl_se, dl_rq); -- cgit v1.2.3 From a3b89864554bbce1594b7abdb5739fc708c1ca95 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 29 Apr 2020 17:05:15 +0200 Subject: rbtree, perf: Use new rbtree helpers Reduce rbtree boiler plate by using the new helpers. One noteworthy change is unification of the various (partial) compare functions. We construct a subtree match by forcing the sub-order to always match, see __group_cmp(). Due to 'const' we had to touch cgroup_id(). Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Acked-by: Tejun Heo Acked-by: Davidlohr Bueso --- include/linux/cgroup.h | 4 +- kernel/events/core.c | 195 +++++++++++++++++++++++-------------------------- 2 files changed, 92 insertions(+), 107 deletions(-) (limited to 'include/linux') diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 451c2d26a5db..4f2f79de083e 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -307,7 +307,7 @@ void css_task_iter_end(struct css_task_iter *it); * Inline functions. */ -static inline u64 cgroup_id(struct cgroup *cgrp) +static inline u64 cgroup_id(const struct cgroup *cgrp) { return cgrp->kn->id; } @@ -701,7 +701,7 @@ void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen); struct cgroup_subsys_state; struct cgroup; -static inline u64 cgroup_id(struct cgroup *cgrp) { return 1; } +static inline u64 cgroup_id(const struct cgroup *cgrp) { return 1; } static inline void css_get(struct cgroup_subsys_state *css) {} static inline void css_put(struct cgroup_subsys_state *css) {} static inline int cgroup_attach_task_all(struct task_struct *from, diff --git a/kernel/events/core.c b/kernel/events/core.c index 55d18791a72d..3d890961f6e5 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -1595,50 +1595,91 @@ static void perf_event_groups_init(struct perf_event_groups *groups) groups->index = 0; } +static inline struct cgroup *event_cgroup(const struct perf_event *event) +{ + struct cgroup *cgroup = NULL; + +#ifdef CONFIG_CGROUP_PERF + if (event->cgrp) + cgroup = event->cgrp->css.cgroup; +#endif + + return cgroup; +} + /* * Compare function for event groups; * * Implements complex key that first sorts by CPU and then by virtual index * which provides ordering when rotating groups for the same CPU. */ -static bool -perf_event_groups_less(struct perf_event *left, struct perf_event *right) +static __always_inline int +perf_event_groups_cmp(const int left_cpu, const struct cgroup *left_cgroup, + const u64 left_group_index, const struct perf_event *right) { - if (left->cpu < right->cpu) - return true; - if (left->cpu > right->cpu) - return false; + if (left_cpu < right->cpu) + return -1; + if (left_cpu > right->cpu) + return 1; #ifdef CONFIG_CGROUP_PERF - if (left->cgrp != right->cgrp) { - if (!left->cgrp || !left->cgrp->css.cgroup) { - /* - * Left has no cgroup but right does, no cgroups come - * first. - */ - return true; - } - if (!right->cgrp || !right->cgrp->css.cgroup) { - /* - * Right has no cgroup but left does, no cgroups come - * first. - */ - return false; - } - /* Two dissimilar cgroups, order by id. */ - if (left->cgrp->css.cgroup->kn->id < right->cgrp->css.cgroup->kn->id) - return true; + { + const struct cgroup *right_cgroup = event_cgroup(right); - return false; + if (left_cgroup != right_cgroup) { + if (!left_cgroup) { + /* + * Left has no cgroup but right does, no + * cgroups come first. + */ + return -1; + } + if (!right_cgroup) { + /* + * Right has no cgroup but left does, no + * cgroups come first. + */ + return 1; + } + /* Two dissimilar cgroups, order by id. */ + if (cgroup_id(left_cgroup) < cgroup_id(right_cgroup)) + return -1; + + return 1; + } } #endif - if (left->group_index < right->group_index) - return true; - if (left->group_index > right->group_index) - return false; + if (left_group_index < right->group_index) + return -1; + if (left_group_index > right->group_index) + return 1; - return false; + return 0; +} + +#define __node_2_pe(node) \ + rb_entry((node), struct perf_event, group_node) + +static inline bool __group_less(struct rb_node *a, const struct rb_node *b) +{ + struct perf_event *e = __node_2_pe(a); + return perf_event_groups_cmp(e->cpu, event_cgroup(e), e->group_index, + __node_2_pe(b)) < 0; +} + +struct __group_key { + int cpu; + struct cgroup *cgroup; +}; + +static inline int __group_cmp(const void *key, const struct rb_node *node) +{ + const struct __group_key *a = key; + const struct perf_event *b = __node_2_pe(node); + + /* partial/subtree match: @cpu, @cgroup; ignore: @group_index */ + return perf_event_groups_cmp(a->cpu, a->cgroup, b->group_index, b); } /* @@ -1650,27 +1691,9 @@ static void perf_event_groups_insert(struct perf_event_groups *groups, struct perf_event *event) { - struct perf_event *node_event; - struct rb_node *parent; - struct rb_node **node; - event->group_index = ++groups->index; - node = &groups->tree.rb_node; - parent = *node; - - while (*node) { - parent = *node; - node_event = container_of(*node, struct perf_event, group_node); - - if (perf_event_groups_less(event, node_event)) - node = &parent->rb_left; - else - node = &parent->rb_right; - } - - rb_link_node(&event->group_node, parent, node); - rb_insert_color(&event->group_node, &groups->tree); + rb_add(&event->group_node, &groups->tree, __group_less); } /* @@ -1718,45 +1741,17 @@ static struct perf_event * perf_event_groups_first(struct perf_event_groups *groups, int cpu, struct cgroup *cgrp) { - struct perf_event *node_event = NULL, *match = NULL; - struct rb_node *node = groups->tree.rb_node; -#ifdef CONFIG_CGROUP_PERF - u64 node_cgrp_id, cgrp_id = 0; - - if (cgrp) - cgrp_id = cgrp->kn->id; -#endif - - while (node) { - node_event = container_of(node, struct perf_event, group_node); - - if (cpu < node_event->cpu) { - node = node->rb_left; - continue; - } - if (cpu > node_event->cpu) { - node = node->rb_right; - continue; - } -#ifdef CONFIG_CGROUP_PERF - node_cgrp_id = 0; - if (node_event->cgrp && node_event->cgrp->css.cgroup) - node_cgrp_id = node_event->cgrp->css.cgroup->kn->id; + struct __group_key key = { + .cpu = cpu, + .cgroup = cgrp, + }; + struct rb_node *node; - if (cgrp_id < node_cgrp_id) { - node = node->rb_left; - continue; - } - if (cgrp_id > node_cgrp_id) { - node = node->rb_right; - continue; - } -#endif - match = node_event; - node = node->rb_left; - } + node = rb_find_first(&key, &groups->tree, __group_cmp); + if (node) + return __node_2_pe(node); - return match; + return NULL; } /* @@ -1765,27 +1760,17 @@ perf_event_groups_first(struct perf_event_groups *groups, int cpu, static struct perf_event * perf_event_groups_next(struct perf_event *event) { - struct perf_event *next; -#ifdef CONFIG_CGROUP_PERF - u64 curr_cgrp_id = 0; - u64 next_cgrp_id = 0; -#endif - - next = rb_entry_safe(rb_next(&event->group_node), typeof(*event), group_node); - if (next == NULL || next->cpu != event->cpu) - return NULL; - -#ifdef CONFIG_CGROUP_PERF - if (event->cgrp && event->cgrp->css.cgroup) - curr_cgrp_id = event->cgrp->css.cgroup->kn->id; + struct __group_key key = { + .cpu = event->cpu, + .cgroup = event_cgroup(event), + }; + struct rb_node *next; - if (next->cgrp && next->cgrp->css.cgroup) - next_cgrp_id = next->cgrp->css.cgroup->kn->id; + next = rb_next_match(&key, &event->group_node, __group_cmp); + if (next) + return __node_2_pe(next); - if (curr_cgrp_id != next_cgrp_id) - return NULL; -#endif - return next; + return NULL; } /* -- cgit v1.2.3 From ae18ad281e825993d190073d0ae2ea35dee27ee1 Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Thu, 28 Jan 2021 14:10:38 +0100 Subject: sched: Remove MAX_USER_RT_PRIO Commit d46523ea32a7 ("[PATCH] fix MAX_USER_RT_PRIO and MAX_RT_PRIO") was introduced due to a a small time period in which the realtime patch set was using different values for MAX_USER_RT_PRIO and MAX_RT_PRIO. This is no longer true, i.e. now MAX_RT_PRIO == MAX_USER_RT_PRIO. Get rid of MAX_USER_RT_PRIO and make everything use MAX_RT_PRIO instead. Signed-off-by: Dietmar Eggemann Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lkml.kernel.org/r/20210128131040.296856-2-dietmar.eggemann@arm.com --- include/linux/sched/prio.h | 9 +-------- kernel/sched/core.c | 7 +++---- 2 files changed, 4 insertions(+), 12 deletions(-) (limited to 'include/linux') diff --git a/include/linux/sched/prio.h b/include/linux/sched/prio.h index 7d64feafc408..d111f2fd77ea 100644 --- a/include/linux/sched/prio.h +++ b/include/linux/sched/prio.h @@ -11,16 +11,9 @@ * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH * tasks are in the range MAX_RT_PRIO..MAX_PRIO-1. Priority * values are inverted: lower p->prio value means higher priority. - * - * The MAX_USER_RT_PRIO value allows the actual maximum - * RT priority to be separate from the value exported to - * user-space. This allows kernel threads to set their - * priority to a value higher than any user task. Note: - * MAX_RT_PRIO must not be smaller than MAX_USER_RT_PRIO. */ -#define MAX_USER_RT_PRIO 100 -#define MAX_RT_PRIO MAX_USER_RT_PRIO +#define MAX_RT_PRIO 100 #define MAX_PRIO (MAX_RT_PRIO + NICE_WIDTH) #define DEFAULT_PRIO (MAX_RT_PRIO + NICE_WIDTH / 2) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 6c789dcb8e5a..f0b0b67fcdee 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5911,11 +5911,10 @@ recheck: /* * Valid priorities for SCHED_FIFO and SCHED_RR are - * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL, + * 1..MAX_RT_PRIO-1, valid priority for SCHED_NORMAL, * SCHED_BATCH and SCHED_IDLE is 0. */ - if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || - (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) + if (attr->sched_priority > MAX_RT_PRIO-1) return -EINVAL; if ((dl_policy(policy) && !__checkparam_dl(attr)) || (rt_policy(policy) != (attr->sched_priority != 0))) @@ -6983,7 +6982,7 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy) switch (policy) { case SCHED_FIFO: case SCHED_RR: - ret = MAX_USER_RT_PRIO-1; + ret = MAX_RT_PRIO-1; break; case SCHED_DEADLINE: case SCHED_NORMAL: -- cgit v1.2.3 From 9d061ba6bc170045857f3efe0bba5def30188d4d Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Thu, 28 Jan 2021 14:10:39 +0100 Subject: sched: Remove USER_PRIO, TASK_USER_PRIO and MAX_USER_PRIO The only remaining use of MAX_USER_PRIO (and USER_PRIO) is the SCALE_PRIO() definition in the PowerPC Cell architecture's Synergistic Processor Unit (SPU) scheduler. TASK_USER_PRIO isn't used anymore. Commit fe443ef2ac42 ("[POWERPC] spusched: Dynamic timeslicing for SCHED_OTHER") copied SCALE_PRIO() from the task scheduler in v2.6.23. Commit a4ec24b48dde ("sched: tidy up SCHED_RR") removed it from the task scheduler in v2.6.24. Commit 3ee237dddcd8 ("sched/prio: Add 3 macros of MAX_NICE, MIN_NICE and NICE_WIDTH in prio.h") introduced NICE_WIDTH much later. With: MAX_USER_PRIO = USER_PRIO(MAX_PRIO) = MAX_PRIO - MAX_RT_PRIO MAX_PRIO = MAX_RT_PRIO + NICE_WIDTH MAX_USER_PRIO = MAX_RT_PRIO + NICE_WIDTH - MAX_RT_PRIO MAX_USER_PRIO = NICE_WIDTH MAX_USER_PRIO can be replaced by NICE_WIDTH to be able to remove all the {*_}USER_PRIO defines. Signed-off-by: Dietmar Eggemann Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lkml.kernel.org/r/20210128131040.296856-3-dietmar.eggemann@arm.com --- arch/powerpc/platforms/cell/spufs/sched.c | 2 +- include/linux/sched/prio.h | 9 --------- kernel/sched/sched.h | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/platforms/cell/spufs/sched.c b/arch/powerpc/platforms/cell/spufs/sched.c index f18d5067cd0f..aeb7f3922106 100644 --- a/arch/powerpc/platforms/cell/spufs/sched.c +++ b/arch/powerpc/platforms/cell/spufs/sched.c @@ -72,7 +72,7 @@ static struct timer_list spuloadavg_timer; #define DEF_SPU_TIMESLICE (100 * HZ / (1000 * SPUSCHED_TICK)) #define SCALE_PRIO(x, prio) \ - max(x * (MAX_PRIO - prio) / (MAX_USER_PRIO / 2), MIN_SPU_TIMESLICE) + max(x * (MAX_PRIO - prio) / (NICE_WIDTH / 2), MIN_SPU_TIMESLICE) /* * scale user-nice values [ -20 ... 0 ... 19 ] to time slice values: diff --git a/include/linux/sched/prio.h b/include/linux/sched/prio.h index d111f2fd77ea..ab83d85e1183 100644 --- a/include/linux/sched/prio.h +++ b/include/linux/sched/prio.h @@ -26,15 +26,6 @@ #define NICE_TO_PRIO(nice) ((nice) + DEFAULT_PRIO) #define PRIO_TO_NICE(prio) ((prio) - DEFAULT_PRIO) -/* - * 'User priority' is the nice value converted to something we - * can work with better when scaling various scheduler parameters, - * it's a [ 0 ... 39 ] range. - */ -#define USER_PRIO(p) ((p)-MAX_RT_PRIO) -#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio) -#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO)) - /* * Convert nice value [19,-20] to rlimit style value [1,40]. */ diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f519aba2c542..2185b3b435a9 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -140,7 +140,7 @@ extern void call_trace_sched_update_nr_running(struct rq *rq, int count); * scale_load() and scale_load_down(w) to convert between them. The * following must be true: * - * scale_load(sched_prio_to_weight[USER_PRIO(NICE_TO_PRIO(0))]) == NICE_0_LOAD + * scale_load(sched_prio_to_weight[NICE_TO_PRIO(0)-MAX_RT_PRIO]) == NICE_0_LOAD * */ #define NICE_0_LOAD (1L << NICE_0_LOAD_SHIFT) -- cgit v1.2.3 From 880cfed3a012d7863f42251791cea7fe78c39390 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 18 Jan 2021 15:12:18 +0100 Subject: static_call: Pull some static_call declarations to the type headers Some static call declarations are going to be needed on low level header files. Move the necessary material to the dedicated static call types header to avoid inclusion dependency hell. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Frederic Weisbecker Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lkml.kernel.org/r/20210118141223.123667-4-frederic@kernel.org --- include/linux/static_call.h | 21 --------------------- include/linux/static_call_types.h | 27 +++++++++++++++++++++++++++ tools/include/linux/static_call_types.h | 27 +++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 21 deletions(-) (limited to 'include/linux') diff --git a/include/linux/static_call.h b/include/linux/static_call.h index 695da4c9b338..a2c064585c03 100644 --- a/include/linux/static_call.h +++ b/include/linux/static_call.h @@ -107,26 +107,10 @@ extern void arch_static_call_transform(void *site, void *tramp, void *func, bool #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name) -/* - * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from - * the symbol table so that objtool can reference it when it generates the - * .static_call_sites section. - */ -#define __static_call(name) \ -({ \ - __ADDRESSABLE(STATIC_CALL_KEY(name)); \ - &STATIC_CALL_TRAMP(name); \ -}) - #else #define STATIC_CALL_TRAMP_ADDR(name) NULL #endif - -#define DECLARE_STATIC_CALL(name, func) \ - extern struct static_call_key STATIC_CALL_KEY(name); \ - extern typeof(func) STATIC_CALL_TRAMP(name); - #define static_call_update(name, func) \ ({ \ BUILD_BUG_ON(!__same_type(*(func), STATIC_CALL_TRAMP(name))); \ @@ -174,7 +158,6 @@ extern int static_call_text_reserved(void *start, void *end); }; \ ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) -#define static_call(name) __static_call(name) #define static_call_cond(name) (void)__static_call(name) #define EXPORT_STATIC_CALL(name) \ @@ -207,7 +190,6 @@ struct static_call_key { }; \ ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) -#define static_call(name) __static_call(name) #define static_call_cond(name) (void)__static_call(name) static inline @@ -252,9 +234,6 @@ struct static_call_key { .func = NULL, \ } -#define static_call(name) \ - ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func)) - static inline void __static_call_nop(void) { } /* diff --git a/include/linux/static_call_types.h b/include/linux/static_call_types.h index 89135bb35bf7..08f78b1b88b4 100644 --- a/include/linux/static_call_types.h +++ b/include/linux/static_call_types.h @@ -4,6 +4,7 @@ #include #include +#include #define STATIC_CALL_KEY_PREFIX __SCK__ #define STATIC_CALL_KEY_PREFIX_STR __stringify(STATIC_CALL_KEY_PREFIX) @@ -32,4 +33,30 @@ struct static_call_site { s32 key; }; +#define DECLARE_STATIC_CALL(name, func) \ + extern struct static_call_key STATIC_CALL_KEY(name); \ + extern typeof(func) STATIC_CALL_TRAMP(name); + +#ifdef CONFIG_HAVE_STATIC_CALL + +/* + * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from + * the symbol table so that objtool can reference it when it generates the + * .static_call_sites section. + */ +#define __static_call(name) \ +({ \ + __ADDRESSABLE(STATIC_CALL_KEY(name)); \ + &STATIC_CALL_TRAMP(name); \ +}) + +#define static_call(name) __static_call(name) + +#else + +#define static_call(name) \ + ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func)) + +#endif /* CONFIG_HAVE_STATIC_CALL */ + #endif /* _STATIC_CALL_TYPES_H */ diff --git a/tools/include/linux/static_call_types.h b/tools/include/linux/static_call_types.h index 89135bb35bf7..08f78b1b88b4 100644 --- a/tools/include/linux/static_call_types.h +++ b/tools/include/linux/static_call_types.h @@ -4,6 +4,7 @@ #include #include +#include #define STATIC_CALL_KEY_PREFIX __SCK__ #define STATIC_CALL_KEY_PREFIX_STR __stringify(STATIC_CALL_KEY_PREFIX) @@ -32,4 +33,30 @@ struct static_call_site { s32 key; }; +#define DECLARE_STATIC_CALL(name, func) \ + extern struct static_call_key STATIC_CALL_KEY(name); \ + extern typeof(func) STATIC_CALL_TRAMP(name); + +#ifdef CONFIG_HAVE_STATIC_CALL + +/* + * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from + * the symbol table so that objtool can reference it when it generates the + * .static_call_sites section. + */ +#define __static_call(name) \ +({ \ + __ADDRESSABLE(STATIC_CALL_KEY(name)); \ + &STATIC_CALL_TRAMP(name); \ +}) + +#define static_call(name) __static_call(name) + +#else + +#define static_call(name) \ + ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func)) + +#endif /* CONFIG_HAVE_STATIC_CALL */ + #endif /* _STATIC_CALL_TYPES_H */ -- cgit v1.2.3 From 3f2a8fc4b15de18644e8a80a09edda168676e22c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 18 Jan 2021 15:12:16 +0100 Subject: static_call/x86: Add __static_call_return0() Provide a stub function that return 0 and wire up the static call site patching to replace the CALL with a single 5 byte instruction that clears %RAX, the return value register. The function can be cast to any function pointer type that has a single %RAX return (including pointers). Also provide a version that returns an int for convenience. We are clearing the entire %RAX register in any case, whether the return value is 32 or 64 bits, since %RAX is always a scratch register anyway. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Frederic Weisbecker Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lkml.kernel.org/r/20210118141223.123667-2-frederic@kernel.org --- arch/x86/kernel/static_call.c | 17 +++++++++++++++-- include/linux/static_call.h | 12 ++++++++++++ kernel/static_call.c | 5 +++++ 3 files changed, 32 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c index ca9a380d9c0b..9442c4136c38 100644 --- a/arch/x86/kernel/static_call.c +++ b/arch/x86/kernel/static_call.c @@ -11,14 +11,26 @@ enum insn_type { RET = 3, /* tramp / site cond-tail-call */ }; +/* + * data16 data16 xorq %rax, %rax - a single 5 byte instruction that clears %rax + * The REX.W cancels the effect of any data16. + */ +static const u8 xor5rax[] = { 0x66, 0x66, 0x48, 0x31, 0xc0 }; + static void __ref __static_call_transform(void *insn, enum insn_type type, void *func) { + const void *emulate = NULL; int size = CALL_INSN_SIZE; const void *code; switch (type) { case CALL: code = text_gen_insn(CALL_INSN_OPCODE, insn, func); + if (func == &__static_call_return0) { + emulate = code; + code = &xor5rax; + } + break; case NOP: @@ -41,7 +53,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, void if (unlikely(system_state == SYSTEM_BOOTING)) return text_poke_early(insn, code, size); - text_poke_bp(insn, code, size, NULL); + text_poke_bp(insn, code, size, emulate); } static void __static_call_validate(void *insn, bool tail) @@ -54,7 +66,8 @@ static void __static_call_validate(void *insn, bool tail) return; } else { if (opcode == CALL_INSN_OPCODE || - !memcmp(insn, ideal_nops[NOP_ATOMIC5], 5)) + !memcmp(insn, ideal_nops[NOP_ATOMIC5], 5) || + !memcmp(insn, xor5rax, 5)) return; } diff --git a/include/linux/static_call.h b/include/linux/static_call.h index a2c064585c03..bd6735d10e2e 100644 --- a/include/linux/static_call.h +++ b/include/linux/static_call.h @@ -142,6 +142,8 @@ extern void __static_call_update(struct static_call_key *key, void *tramp, void extern int static_call_mod_init(struct module *mod); extern int static_call_text_reserved(void *start, void *end); +extern long __static_call_return0(void); + #define DEFINE_STATIC_CALL(name, _func) \ DECLARE_STATIC_CALL(name, _func); \ struct static_call_key STATIC_CALL_KEY(name) = { \ @@ -206,6 +208,11 @@ static inline int static_call_text_reserved(void *start, void *end) return 0; } +static inline long __static_call_return0(void) +{ + return 0; +} + #define EXPORT_STATIC_CALL(name) \ EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) @@ -222,6 +229,11 @@ struct static_call_key { void *func; }; +static inline long __static_call_return0(void) +{ + return 0; +} + #define DEFINE_STATIC_CALL(name, _func) \ DECLARE_STATIC_CALL(name, _func); \ struct static_call_key STATIC_CALL_KEY(name) = { \ diff --git a/kernel/static_call.c b/kernel/static_call.c index 84565c2a41b8..0bc11b5ce681 100644 --- a/kernel/static_call.c +++ b/kernel/static_call.c @@ -438,6 +438,11 @@ int __init static_call_init(void) } early_initcall(static_call_init); +long __static_call_return0(void) +{ + return 0; +} + #ifdef CONFIG_STATIC_CALL_SELFTEST static int func_a(int x) -- cgit v1.2.3 From 29fd01944b7273bb630c649a2104b7f9e4ef3fa6 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 18 Jan 2021 15:12:17 +0100 Subject: static_call: Provide DEFINE_STATIC_CALL_RET0() DECLARE_STATIC_CALL() must pass the original function targeted for a given static call. But DEFINE_STATIC_CALL() may want to initialize it as off. In this case we can't pass NULL (for functions without return value) or __static_call_return0 (for functions returning a value) directly to DEFINE_STATIC_CALL() as that may trigger a static call redeclaration with a different function prototype. Type casts neither can work around that as they don't get along with typeof(). The proper way to do that for functions that don't return a value is to use DEFINE_STATIC_CALL_NULL(). But functions returning a actual value don't have an equivalent yet. Provide DEFINE_STATIC_CALL_RET0() to solve this situation. Signed-off-by: Frederic Weisbecker Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lkml.kernel.org/r/20210118141223.123667-3-frederic@kernel.org --- include/linux/static_call.h | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/static_call.h b/include/linux/static_call.h index bd6735d10e2e..d69dd8b976ca 100644 --- a/include/linux/static_call.h +++ b/include/linux/static_call.h @@ -144,13 +144,13 @@ extern int static_call_text_reserved(void *start, void *end); extern long __static_call_return0(void); -#define DEFINE_STATIC_CALL(name, _func) \ +#define __DEFINE_STATIC_CALL(name, _func, _func_init) \ DECLARE_STATIC_CALL(name, _func); \ struct static_call_key STATIC_CALL_KEY(name) = { \ - .func = _func, \ + .func = _func_init, \ .type = 1, \ }; \ - ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) + ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func_init) #define DEFINE_STATIC_CALL_NULL(name, _func) \ DECLARE_STATIC_CALL(name, _func); \ @@ -178,12 +178,12 @@ struct static_call_key { void *func; }; -#define DEFINE_STATIC_CALL(name, _func) \ +#define __DEFINE_STATIC_CALL(name, _func, _func_init) \ DECLARE_STATIC_CALL(name, _func); \ struct static_call_key STATIC_CALL_KEY(name) = { \ - .func = _func, \ + .func = _func_init, \ }; \ - ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) + ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func_init) #define DEFINE_STATIC_CALL_NULL(name, _func) \ DECLARE_STATIC_CALL(name, _func); \ @@ -234,10 +234,10 @@ static inline long __static_call_return0(void) return 0; } -#define DEFINE_STATIC_CALL(name, _func) \ +#define __DEFINE_STATIC_CALL(name, _func, _func_init) \ DECLARE_STATIC_CALL(name, _func); \ struct static_call_key STATIC_CALL_KEY(name) = { \ - .func = _func, \ + .func = _func_init, \ } #define DEFINE_STATIC_CALL_NULL(name, _func) \ @@ -286,4 +286,10 @@ static inline int static_call_text_reserved(void *start, void *end) #endif /* CONFIG_HAVE_STATIC_CALL */ +#define DEFINE_STATIC_CALL(name, _func) \ + __DEFINE_STATIC_CALL(name, _func, _func) + +#define DEFINE_STATIC_CALL_RET0(name, _func) \ + __DEFINE_STATIC_CALL(name, _func, __static_call_return0) + #endif /* _LINUX_STATIC_CALL_H */ -- cgit v1.2.3 From b965f1ddb47daa5b8b2e2bc9c921431236830367 Mon Sep 17 00:00:00 2001 From: "Peter Zijlstra (Intel)" Date: Mon, 18 Jan 2021 15:12:20 +0100 Subject: preempt/dynamic: Provide cond_resched() and might_resched() static calls Provide static calls to control cond_resched() (called in !CONFIG_PREEMPT) and might_resched() (called in CONFIG_PREEMPT_VOLUNTARY) to that we can override their behaviour when preempt= is overriden. Since the default behaviour is full preemption, both their calls are ignored when preempt= isn't passed. [fweisbec: branch might_resched() directly to __cond_resched(), only define static calls when PREEMPT_DYNAMIC] Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Frederic Weisbecker Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lkml.kernel.org/r/20210118141223.123667-6-frederic@kernel.org --- include/linux/kernel.h | 23 +++++++++++++++++++---- include/linux/sched.h | 27 ++++++++++++++++++++++++--- kernel/sched/core.c | 16 +++++++++++++--- 3 files changed, 56 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kernel.h b/include/linux/kernel.h index f7902d8c1048..cfd3d349f905 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -15,7 +15,7 @@ #include #include #include - +#include #include #include @@ -81,11 +81,26 @@ struct pt_regs; struct user; #ifdef CONFIG_PREEMPT_VOLUNTARY -extern int _cond_resched(void); -# define might_resched() _cond_resched() + +extern int __cond_resched(void); +# define might_resched() __cond_resched() + +#elif defined(CONFIG_PREEMPT_DYNAMIC) + +extern int __cond_resched(void); + +DECLARE_STATIC_CALL(might_resched, __cond_resched); + +static __always_inline void might_resched(void) +{ + static_call(might_resched)(); +} + #else + # define might_resched() do { } while (0) -#endif + +#endif /* CONFIG_PREEMPT_* */ #ifdef CONFIG_DEBUG_ATOMIC_SLEEP extern void ___might_sleep(const char *file, int line, int preempt_offset); diff --git a/include/linux/sched.h b/include/linux/sched.h index e1152227bb79..2f35594b8b53 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1871,11 +1871,32 @@ static inline int test_tsk_need_resched(struct task_struct *tsk) * value indicates whether a reschedule was done in fact. * cond_resched_lock() will drop the spinlock before scheduling, */ -#ifndef CONFIG_PREEMPTION -extern int _cond_resched(void); +#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) +extern int __cond_resched(void); + +#ifdef CONFIG_PREEMPT_DYNAMIC + +DECLARE_STATIC_CALL(cond_resched, __cond_resched); + +static __always_inline int _cond_resched(void) +{ + return static_call(cond_resched)(); +} + #else + +static inline int _cond_resched(void) +{ + return __cond_resched(); +} + +#endif /* CONFIG_PREEMPT_DYNAMIC */ + +#else + static inline int _cond_resched(void) { return 0; } -#endif + +#endif /* !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) */ #define cond_resched() ({ \ ___might_sleep(__FILE__, __LINE__, 0); \ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 4afbdd27f46d..f7c8fd8fa177 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6785,17 +6785,27 @@ SYSCALL_DEFINE0(sched_yield) return 0; } -#ifndef CONFIG_PREEMPTION -int __sched _cond_resched(void) +#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) +int __sched __cond_resched(void) { if (should_resched(0)) { preempt_schedule_common(); return 1; } +#ifndef CONFIG_PREEMPT_RCU rcu_all_qs(); +#endif return 0; } -EXPORT_SYMBOL(_cond_resched); +EXPORT_SYMBOL(__cond_resched); +#endif + +#ifdef CONFIG_PREEMPT_DYNAMIC +DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); +EXPORT_STATIC_CALL(cond_resched); + +DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); +EXPORT_STATIC_CALL(might_resched); #endif /* -- cgit v1.2.3 From 40607ee97e4eec5655cc0f76a720bdc4c63a6434 Mon Sep 17 00:00:00 2001 From: "Peter Zijlstra (Intel)" Date: Mon, 18 Jan 2021 15:12:22 +0100 Subject: preempt/dynamic: Provide irqentry_exit_cond_resched() static call Provide static call to control IRQ preemption (called in CONFIG_PREEMPT) so that we can override its behaviour when preempt= is overriden. Since the default behaviour is full preemption, its call is initialized to provide IRQ preemption when preempt= isn't passed. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Frederic Weisbecker Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lkml.kernel.org/r/20210118141223.123667-8-frederic@kernel.org --- include/linux/entry-common.h | 4 ++++ kernel/entry/common.c | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h index a104b298019a..883acef895bc 100644 --- a/include/linux/entry-common.h +++ b/include/linux/entry-common.h @@ -2,6 +2,7 @@ #ifndef __LINUX_ENTRYCOMMON_H #define __LINUX_ENTRYCOMMON_H +#include #include #include #include @@ -454,6 +455,9 @@ irqentry_state_t noinstr irqentry_enter(struct pt_regs *regs); * Conditional reschedule with additional sanity checks. */ void irqentry_exit_cond_resched(void); +#ifdef CONFIG_PREEMPT_DYNAMIC +DECLARE_STATIC_CALL(irqentry_exit_cond_resched, irqentry_exit_cond_resched); +#endif /** * irqentry_exit - Handle return from exception that used irqentry_enter() diff --git a/kernel/entry/common.c b/kernel/entry/common.c index f9d491b17b78..f09cae37ddd5 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -385,6 +385,9 @@ void irqentry_exit_cond_resched(void) preempt_schedule_irq(); } } +#ifdef CONFIG_PREEMPT_DYNAMIC +DEFINE_STATIC_CALL(irqentry_exit_cond_resched, irqentry_exit_cond_resched); +#endif noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) { @@ -411,8 +414,13 @@ noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) } instrumentation_begin(); - if (IS_ENABLED(CONFIG_PREEMPTION)) + if (IS_ENABLED(CONFIG_PREEMPTION)) { +#ifdef CONFIG_PREEMT_DYNAMIC + static_call(irqentry_exit_cond_resched)(); +#else irqentry_exit_cond_resched(); +#endif + } /* Covers both tracing and lockdep */ trace_hardirqs_on(); instrumentation_end(); -- cgit v1.2.3 From 73f44fe19d359635a607e8e8daa0da4001c1cfc2 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 27 Jan 2021 17:18:37 -0600 Subject: static_call: Allow module use without exposing static_call_key When exporting static_call_key; with EXPORT_STATIC_CALL*(), the module can use static_call_update() to change the function called. This is not desirable in general. Not exporting static_call_key however also disallows usage of static_call(), since objtool needs the key to construct the static_call_site. Solve this by allowing objtool to create the static_call_site using the trampoline address when it builds a module and cannot find the static_call_key symbol. The module loader will then try and map the trampole back to a key before it constructs the normal sites list. Doing this requires a trampoline -> key associsation, so add another magic section that keeps those. Originally-by: Peter Zijlstra (Intel) Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lkml.kernel.org/r/20210127231837.ifddpn7rhwdaepiu@treble --- arch/x86/include/asm/static_call.h | 7 +++++ include/asm-generic/vmlinux.lds.h | 5 ++- include/linux/static_call.h | 22 +++++++++++-- include/linux/static_call_types.h | 27 ++++++++++++++-- kernel/static_call.c | 55 +++++++++++++++++++++++++++++++-- tools/include/linux/static_call_types.h | 27 ++++++++++++++-- tools/objtool/check.c | 17 ++++++++-- 7 files changed, 149 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/static_call.h b/arch/x86/include/asm/static_call.h index c37f11999d0c..cbb67b6030f9 100644 --- a/arch/x86/include/asm/static_call.h +++ b/arch/x86/include/asm/static_call.h @@ -37,4 +37,11 @@ #define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) \ __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; nop; nop; nop; nop") + +#define ARCH_ADD_TRAMP_KEY(name) \ + asm(".pushsection .static_call_tramp_key, \"a\" \n" \ + ".long " STATIC_CALL_TRAMP_STR(name) " - . \n" \ + ".long " STATIC_CALL_KEY_STR(name) " - . \n" \ + ".popsection \n") + #endif /* _ASM_STATIC_CALL_H */ diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index b97c628ad91f..3f747de1934d 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -393,7 +393,10 @@ . = ALIGN(8); \ __start_static_call_sites = .; \ KEEP(*(.static_call_sites)) \ - __stop_static_call_sites = .; + __stop_static_call_sites = .; \ + __start_static_call_tramp_key = .; \ + KEEP(*(.static_call_tramp_key)) \ + __stop_static_call_tramp_key = .; /* * Allow architectures to handle ro_after_init data on their diff --git a/include/linux/static_call.h b/include/linux/static_call.h index d69dd8b976ca..85ecc789f4ff 100644 --- a/include/linux/static_call.h +++ b/include/linux/static_call.h @@ -138,6 +138,12 @@ struct static_call_key { }; }; +/* For finding the key associated with a trampoline */ +struct static_call_tramp_key { + s32 tramp; + s32 key; +}; + extern void __static_call_update(struct static_call_key *key, void *tramp, void *func); extern int static_call_mod_init(struct module *mod); extern int static_call_text_reserved(void *start, void *end); @@ -165,11 +171,18 @@ extern long __static_call_return0(void); #define EXPORT_STATIC_CALL(name) \ EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) - #define EXPORT_STATIC_CALL_GPL(name) \ EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) +/* Leave the key unexported, so modules can't change static call targets: */ +#define EXPORT_STATIC_CALL_TRAMP(name) \ + EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \ + ARCH_ADD_TRAMP_KEY(name) +#define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ + EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \ + ARCH_ADD_TRAMP_KEY(name) + #elif defined(CONFIG_HAVE_STATIC_CALL) static inline int static_call_init(void) { return 0; } @@ -216,11 +229,16 @@ static inline long __static_call_return0(void) #define EXPORT_STATIC_CALL(name) \ EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) - #define EXPORT_STATIC_CALL_GPL(name) \ EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) +/* Leave the key unexported, so modules can't change static call targets: */ +#define EXPORT_STATIC_CALL_TRAMP(name) \ + EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) +#define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ + EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) + #else /* Generic implementation */ static inline int static_call_init(void) { return 0; } diff --git a/include/linux/static_call_types.h b/include/linux/static_call_types.h index 08f78b1b88b4..ae5662d368b9 100644 --- a/include/linux/static_call_types.h +++ b/include/linux/static_call_types.h @@ -10,6 +10,7 @@ #define STATIC_CALL_KEY_PREFIX_STR __stringify(STATIC_CALL_KEY_PREFIX) #define STATIC_CALL_KEY_PREFIX_LEN (sizeof(STATIC_CALL_KEY_PREFIX_STR) - 1) #define STATIC_CALL_KEY(name) __PASTE(STATIC_CALL_KEY_PREFIX, name) +#define STATIC_CALL_KEY_STR(name) __stringify(STATIC_CALL_KEY(name)) #define STATIC_CALL_TRAMP_PREFIX __SCT__ #define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX) @@ -39,17 +40,39 @@ struct static_call_site { #ifdef CONFIG_HAVE_STATIC_CALL +#define __raw_static_call(name) (&STATIC_CALL_TRAMP(name)) + +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE + /* * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from * the symbol table so that objtool can reference it when it generates the * .static_call_sites section. */ +#define __STATIC_CALL_ADDRESSABLE(name) \ + __ADDRESSABLE(STATIC_CALL_KEY(name)) + #define __static_call(name) \ ({ \ - __ADDRESSABLE(STATIC_CALL_KEY(name)); \ - &STATIC_CALL_TRAMP(name); \ + __STATIC_CALL_ADDRESSABLE(name); \ + __raw_static_call(name); \ }) +#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */ + +#define __STATIC_CALL_ADDRESSABLE(name) +#define __static_call(name) __raw_static_call(name) + +#endif /* CONFIG_HAVE_STATIC_CALL_INLINE */ + +#ifdef MODULE +#define __STATIC_CALL_MOD_ADDRESSABLE(name) +#define static_call_mod(name) __raw_static_call(name) +#else +#define __STATIC_CALL_MOD_ADDRESSABLE(name) __STATIC_CALL_ADDRESSABLE(name) +#define static_call_mod(name) __static_call(name) +#endif + #define static_call(name) __static_call(name) #else diff --git a/kernel/static_call.c b/kernel/static_call.c index 0bc11b5ce681..6906c6ec4c97 100644 --- a/kernel/static_call.c +++ b/kernel/static_call.c @@ -12,6 +12,8 @@ extern struct static_call_site __start_static_call_sites[], __stop_static_call_sites[]; +extern struct static_call_tramp_key __start_static_call_tramp_key[], + __stop_static_call_tramp_key[]; static bool static_call_initialized; @@ -323,10 +325,59 @@ static int __static_call_mod_text_reserved(void *start, void *end) return ret; } +static unsigned long tramp_key_lookup(unsigned long addr) +{ + struct static_call_tramp_key *start = __start_static_call_tramp_key; + struct static_call_tramp_key *stop = __stop_static_call_tramp_key; + struct static_call_tramp_key *tramp_key; + + for (tramp_key = start; tramp_key != stop; tramp_key++) { + unsigned long tramp; + + tramp = (long)tramp_key->tramp + (long)&tramp_key->tramp; + if (tramp == addr) + return (long)tramp_key->key + (long)&tramp_key->key; + } + + return 0; +} + static int static_call_add_module(struct module *mod) { - return __static_call_init(mod, mod->static_call_sites, - mod->static_call_sites + mod->num_static_call_sites); + struct static_call_site *start = mod->static_call_sites; + struct static_call_site *stop = start + mod->num_static_call_sites; + struct static_call_site *site; + + for (site = start; site != stop; site++) { + unsigned long addr = (unsigned long)static_call_key(site); + unsigned long key; + + /* + * Is the key is exported, 'addr' points to the key, which + * means modules are allowed to call static_call_update() on + * it. + * + * Otherwise, the key isn't exported, and 'addr' points to the + * trampoline so we need to lookup the key. + * + * We go through this dance to prevent crazy modules from + * abusing sensitive static calls. + */ + if (!kernel_text_address(addr)) + continue; + + key = tramp_key_lookup(addr); + if (!key) { + pr_warn("Failed to fixup __raw_static_call() usage at: %ps\n", + static_call_addr(site)); + return -EINVAL; + } + + site->key = (key - (long)&site->key) | + (site->key & STATIC_CALL_SITE_FLAGS); + } + + return __static_call_init(mod, start, stop); } static void static_call_del_module(struct module *mod) diff --git a/tools/include/linux/static_call_types.h b/tools/include/linux/static_call_types.h index 08f78b1b88b4..ae5662d368b9 100644 --- a/tools/include/linux/static_call_types.h +++ b/tools/include/linux/static_call_types.h @@ -10,6 +10,7 @@ #define STATIC_CALL_KEY_PREFIX_STR __stringify(STATIC_CALL_KEY_PREFIX) #define STATIC_CALL_KEY_PREFIX_LEN (sizeof(STATIC_CALL_KEY_PREFIX_STR) - 1) #define STATIC_CALL_KEY(name) __PASTE(STATIC_CALL_KEY_PREFIX, name) +#define STATIC_CALL_KEY_STR(name) __stringify(STATIC_CALL_KEY(name)) #define STATIC_CALL_TRAMP_PREFIX __SCT__ #define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX) @@ -39,17 +40,39 @@ struct static_call_site { #ifdef CONFIG_HAVE_STATIC_CALL +#define __raw_static_call(name) (&STATIC_CALL_TRAMP(name)) + +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE + /* * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from * the symbol table so that objtool can reference it when it generates the * .static_call_sites section. */ +#define __STATIC_CALL_ADDRESSABLE(name) \ + __ADDRESSABLE(STATIC_CALL_KEY(name)) + #define __static_call(name) \ ({ \ - __ADDRESSABLE(STATIC_CALL_KEY(name)); \ - &STATIC_CALL_TRAMP(name); \ + __STATIC_CALL_ADDRESSABLE(name); \ + __raw_static_call(name); \ }) +#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */ + +#define __STATIC_CALL_ADDRESSABLE(name) +#define __static_call(name) __raw_static_call(name) + +#endif /* CONFIG_HAVE_STATIC_CALL_INLINE */ + +#ifdef MODULE +#define __STATIC_CALL_MOD_ADDRESSABLE(name) +#define static_call_mod(name) __raw_static_call(name) +#else +#define __STATIC_CALL_MOD_ADDRESSABLE(name) __STATIC_CALL_ADDRESSABLE(name) +#define static_call_mod(name) __static_call(name) +#endif + #define static_call(name) __static_call(name) #else diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 4bd30315eb62..f2e5e5ce1a05 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -502,8 +502,21 @@ static int create_static_call_sections(struct objtool_file *file) key_sym = find_symbol_by_name(file->elf, tmp); if (!key_sym) { - WARN("static_call: can't find static_call_key symbol: %s", tmp); - return -1; + if (!module) { + WARN("static_call: can't find static_call_key symbol: %s", tmp); + return -1; + } + + /* + * For modules(), the key might not be exported, which + * means the module can make static calls but isn't + * allowed to change them. + * + * In that case we temporarily set the key to be the + * trampoline address. This is fixed up in + * static_call_add_module(). + */ + key_sym = insn->call_dest; } free(key_name); -- cgit v1.2.3 From ef72661e28c64ad610f89acc2832ec67b27ba438 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 25 Jan 2021 16:26:50 +0100 Subject: sched: Harden PREEMPT_DYNAMIC Use the new EXPORT_STATIC_CALL_TRAMP() / static_call_mod() to unexport the static_call_key for the PREEMPT_DYNAMIC calls such that modules can no longer update these calls. Having modules change/hi-jack the preemption calls would be horrible. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar --- arch/x86/include/asm/preempt.h | 4 ++-- include/linux/kernel.h | 2 +- include/linux/sched.h | 2 +- kernel/sched/core.c | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h index 9b12dce9bda5..0aa96f824af1 100644 --- a/arch/x86/include/asm/preempt.h +++ b/arch/x86/include/asm/preempt.h @@ -114,7 +114,7 @@ DECLARE_STATIC_CALL(preempt_schedule, __preempt_schedule_func); #define __preempt_schedule() \ do { \ - __ADDRESSABLE(STATIC_CALL_KEY(preempt_schedule)); \ + __STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule); \ asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule) : ASM_CALL_CONSTRAINT); \ } while (0) @@ -127,7 +127,7 @@ DECLARE_STATIC_CALL(preempt_schedule_notrace, __preempt_schedule_notrace_func); #define __preempt_schedule_notrace() \ do { \ - __ADDRESSABLE(STATIC_CALL_KEY(preempt_schedule_notrace)); \ + __STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule_notrace); \ asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule_notrace) : ASM_CALL_CONSTRAINT); \ } while (0) diff --git a/include/linux/kernel.h b/include/linux/kernel.h index cfd3d349f905..5b7ed6dc99ac 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -93,7 +93,7 @@ DECLARE_STATIC_CALL(might_resched, __cond_resched); static __always_inline void might_resched(void) { - static_call(might_resched)(); + static_call_mod(might_resched)(); } #else diff --git a/include/linux/sched.h b/include/linux/sched.h index 2f35594b8b53..4d568288abf9 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1880,7 +1880,7 @@ DECLARE_STATIC_CALL(cond_resched, __cond_resched); static __always_inline int _cond_resched(void) { - return static_call(cond_resched)(); + return static_call_mod(cond_resched)(); } #else diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 4a17bb5f28b0..cec507be460c 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5267,7 +5267,7 @@ EXPORT_SYMBOL(preempt_schedule); #ifdef CONFIG_PREEMPT_DYNAMIC DEFINE_STATIC_CALL(preempt_schedule, __preempt_schedule_func); -EXPORT_STATIC_CALL(preempt_schedule); +EXPORT_STATIC_CALL_TRAMP(preempt_schedule); #endif @@ -5325,7 +5325,7 @@ EXPORT_SYMBOL_GPL(preempt_schedule_notrace); #ifdef CONFIG_PREEMPT_DYNAMIC DEFINE_STATIC_CALL(preempt_schedule_notrace, __preempt_schedule_notrace_func); -EXPORT_STATIC_CALL(preempt_schedule_notrace); +EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); #endif #endif /* CONFIG_PREEMPTION */ @@ -6997,10 +6997,10 @@ EXPORT_SYMBOL(__cond_resched); #ifdef CONFIG_PREEMPT_DYNAMIC DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); -EXPORT_STATIC_CALL(cond_resched); +EXPORT_STATIC_CALL_TRAMP(cond_resched); DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); -EXPORT_STATIC_CALL(might_resched); +EXPORT_STATIC_CALL_TRAMP(might_resched); #endif /* -- cgit v1.2.3 From 43789ef3f7d61aa7bed0cb2764e588fc990c30ef Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 1 Feb 2021 00:05:45 +0100 Subject: rcu/nocb: Perform deferred wake up before last idle's need_resched() check Entering RCU idle mode may cause a deferred wake up of an RCU NOCB_GP kthread (rcuog) to be serviced. Usually a local wake up happening while running the idle task is handled in one of the need_resched() checks carefully placed within the idle loop that can break to the scheduler. Unfortunately the call to rcu_idle_enter() is already beyond the last generic need_resched() check and we may halt the CPU with a resched request unhandled, leaving the task hanging. Fix this with splitting the rcuog wakeup handling from rcu_idle_enter() and place it before the last generic need_resched() check in the idle loop. It is then assumed that no call to call_rcu() will be performed after that in the idle loop until the CPU is put in low power mode. Fixes: 96d3fd0d315a (rcu: Break call_rcu() deadlock involving scheduler and perf) Reported-by: Paul E. McKenney Signed-off-by: Frederic Weisbecker Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: stable@vger.kernel.org Link: https://lkml.kernel.org/r/20210131230548.32970-3-frederic@kernel.org --- include/linux/rcupdate.h | 2 ++ kernel/rcu/tree.c | 3 --- kernel/rcu/tree_plugin.h | 5 +++++ kernel/sched/idle.c | 1 + 4 files changed, 8 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index fd02c5fa60cb..36c2119de702 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -110,8 +110,10 @@ static inline void rcu_user_exit(void) { } #ifdef CONFIG_RCU_NOCB_CPU void rcu_init_nohz(void); +void rcu_nocb_flush_deferred_wakeup(void); #else /* #ifdef CONFIG_RCU_NOCB_CPU */ static inline void rcu_init_nohz(void) { } +static inline void rcu_nocb_flush_deferred_wakeup(void) { } #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ /** diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 63032e5620b9..82838e93b498 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -671,10 +671,7 @@ static noinstr void rcu_eqs_enter(bool user) */ void rcu_idle_enter(void) { - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); - lockdep_assert_irqs_disabled(); - do_nocb_deferred_wakeup(rdp); rcu_eqs_enter(false); } EXPORT_SYMBOL_GPL(rcu_idle_enter); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 7e291ce0a1d6..d5b38c28abd1 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -2187,6 +2187,11 @@ static void do_nocb_deferred_wakeup(struct rcu_data *rdp) do_nocb_deferred_wakeup_common(rdp); } +void rcu_nocb_flush_deferred_wakeup(void) +{ + do_nocb_deferred_wakeup(this_cpu_ptr(&rcu_data)); +} + void __init rcu_init_nohz(void) { int cpu; diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 305727ea0677..7199e6f23789 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -285,6 +285,7 @@ static void do_idle(void) } arch_cpu_idle_enter(); + rcu_nocb_flush_deferred_wakeup(); /* * In poll mode we reenable interrupts and spin. Also if we -- cgit v1.2.3 From 4ae7dc97f726ea95c58ac58af71cc034ad22d7de Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 1 Feb 2021 00:05:48 +0100 Subject: entry/kvm: Explicitly flush pending rcuog wakeup before last rescheduling point Following the idle loop model, cleanly check for pending rcuog wakeup before the last rescheduling point upon resuming to guest mode. This way we can avoid to do it from rcu_user_enter() with the last resort self-IPI hack that enforces rescheduling. Suggested-by: Peter Zijlstra Signed-off-by: Frederic Weisbecker Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: stable@vger.kernel.org Link: https://lkml.kernel.org/r/20210131230548.32970-6-frederic@kernel.org --- arch/x86/kvm/x86.c | 1 + include/linux/entry-kvm.h | 14 ++++++++++++++ kernel/rcu/tree.c | 44 ++++++++++++++++++++++++++++++++++---------- kernel/rcu/tree_plugin.h | 1 + 4 files changed, 50 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 1b404e4d7dd8..b967c1c774a1 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1782,6 +1782,7 @@ EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr); bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu) { + xfer_to_guest_mode_prepare(); return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending(); } diff --git a/include/linux/entry-kvm.h b/include/linux/entry-kvm.h index 9b93f8584ff7..8b2b1d68b954 100644 --- a/include/linux/entry-kvm.h +++ b/include/linux/entry-kvm.h @@ -46,6 +46,20 @@ static inline int arch_xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu, */ int xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu); +/** + * xfer_to_guest_mode_prepare - Perform last minute preparation work that + * need to be handled while IRQs are disabled + * upon entering to guest. + * + * Has to be invoked with interrupts disabled before the last call + * to xfer_to_guest_mode_work_pending(). + */ +static inline void xfer_to_guest_mode_prepare(void) +{ + lockdep_assert_irqs_disabled(); + rcu_nocb_flush_deferred_wakeup(); +} + /** * __xfer_to_guest_mode_work_pending - Check if work is pending * diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 2ebc211fffcb..ce17b8477442 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -678,9 +678,10 @@ EXPORT_SYMBOL_GPL(rcu_idle_enter); #ifdef CONFIG_NO_HZ_FULL +#if !defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK) /* * An empty function that will trigger a reschedule on - * IRQ tail once IRQs get re-enabled on userspace resume. + * IRQ tail once IRQs get re-enabled on userspace/guest resume. */ static void late_wakeup_func(struct irq_work *work) { @@ -689,6 +690,37 @@ static void late_wakeup_func(struct irq_work *work) static DEFINE_PER_CPU(struct irq_work, late_wakeup_work) = IRQ_WORK_INIT(late_wakeup_func); +/* + * If either: + * + * 1) the task is about to enter in guest mode and $ARCH doesn't support KVM generic work + * 2) the task is about to enter in user mode and $ARCH doesn't support generic entry. + * + * In these cases the late RCU wake ups aren't supported in the resched loops and our + * last resort is to fire a local irq_work that will trigger a reschedule once IRQs + * get re-enabled again. + */ +noinstr static void rcu_irq_work_resched(void) +{ + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); + + if (IS_ENABLED(CONFIG_GENERIC_ENTRY) && !(current->flags & PF_VCPU)) + return; + + if (IS_ENABLED(CONFIG_KVM_XFER_TO_GUEST_WORK) && (current->flags & PF_VCPU)) + return; + + instrumentation_begin(); + if (do_nocb_deferred_wakeup(rdp) && need_resched()) { + irq_work_queue(this_cpu_ptr(&late_wakeup_work)); + } + instrumentation_end(); +} + +#else +static inline void rcu_irq_work_resched(void) { } +#endif + /** * rcu_user_enter - inform RCU that we are resuming userspace. * @@ -702,8 +734,6 @@ static DEFINE_PER_CPU(struct irq_work, late_wakeup_work) = */ noinstr void rcu_user_enter(void) { - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); - lockdep_assert_irqs_disabled(); /* @@ -711,13 +741,7 @@ noinstr void rcu_user_enter(void) * rescheduling opportunity in the entry code. Trigger a self IPI * that will fire and reschedule once we resume in user/guest mode. */ - instrumentation_begin(); - if (!IS_ENABLED(CONFIG_GENERIC_ENTRY) || (current->flags & PF_VCPU)) { - if (do_nocb_deferred_wakeup(rdp) && need_resched()) - irq_work_queue(this_cpu_ptr(&late_wakeup_work)); - } - instrumentation_end(); - + rcu_irq_work_resched(); rcu_eqs_enter(true); } diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 384856e4d13e..cdc1b7651c03 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -2197,6 +2197,7 @@ void rcu_nocb_flush_deferred_wakeup(void) { do_nocb_deferred_wakeup(this_cpu_ptr(&rcu_data)); } +EXPORT_SYMBOL_GPL(rcu_nocb_flush_deferred_wakeup); void __init rcu_init_nohz(void) { -- cgit v1.2.3 From 96313e1db8e5629cc2217616dca78f03e6463008 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Wed, 17 Feb 2021 12:25:57 -0800 Subject: net: mdio: Remove of_phy_attach() We have no in-tree users, also update the sfp-phylink.rst documentation to indicate that phy_attach_direct() is used instead of of_phy_attach(). Signed-off-by: Florian Fainelli Reviewed-by: Andrew Lunn Signed-off-by: David S. Miller --- Documentation/networking/sfp-phylink.rst | 2 +- drivers/net/mdio/of_mdio.c | 30 ------------------------------ include/linux/of_mdio.h | 10 ---------- 3 files changed, 1 insertion(+), 41 deletions(-) (limited to 'include/linux') diff --git a/Documentation/networking/sfp-phylink.rst b/Documentation/networking/sfp-phylink.rst index 5aec7c8857d0..328f8d39eeb3 100644 --- a/Documentation/networking/sfp-phylink.rst +++ b/Documentation/networking/sfp-phylink.rst @@ -163,7 +163,7 @@ this documentation. err = phylink_of_phy_connect(priv->phylink, node, flags); For the most part, ``flags`` can be zero; these flags are passed to - the of_phy_attach() inside this function call if a PHY is specified + the phy_attach_direct() inside this function call if a PHY is specified in the DT node ``node``. ``node`` should be the DT node which contains the network phy property, diff --git a/drivers/net/mdio/of_mdio.c b/drivers/net/mdio/of_mdio.c index 4daf94bb56a5..ea9d5855fb52 100644 --- a/drivers/net/mdio/of_mdio.c +++ b/drivers/net/mdio/of_mdio.c @@ -462,36 +462,6 @@ struct phy_device *of_phy_get_and_connect(struct net_device *dev, } EXPORT_SYMBOL(of_phy_get_and_connect); -/** - * of_phy_attach - Attach to a PHY without starting the state machine - * @dev: pointer to net_device claiming the phy - * @phy_np: Node pointer for the PHY - * @flags: flags to pass to the PHY - * @iface: PHY data interface type - * - * If successful, returns a pointer to the phy_device with the embedded - * struct device refcount incremented by one, or NULL on failure. The - * refcount must be dropped by calling phy_disconnect() or phy_detach(). - */ -struct phy_device *of_phy_attach(struct net_device *dev, - struct device_node *phy_np, u32 flags, - phy_interface_t iface) -{ - struct phy_device *phy = of_phy_find_device(phy_np); - int ret; - - if (!phy) - return NULL; - - ret = phy_attach_direct(dev, phy, flags, iface); - - /* refcount is held by phy_attach_direct() on success */ - put_device(&phy->mdio.dev); - - return ret ? NULL : phy; -} -EXPORT_SYMBOL(of_phy_attach); - /* * of_phy_is_fixed_link() and of_phy_register_fixed_link() must * support two DT bindings: diff --git a/include/linux/of_mdio.h b/include/linux/of_mdio.h index cfe8c607a628..2b05e7f7c238 100644 --- a/include/linux/of_mdio.h +++ b/include/linux/of_mdio.h @@ -26,9 +26,6 @@ of_phy_connect(struct net_device *dev, struct device_node *phy_np, struct phy_device * of_phy_get_and_connect(struct net_device *dev, struct device_node *np, void (*hndlr)(struct net_device *)); -struct phy_device * -of_phy_attach(struct net_device *dev, struct device_node *phy_np, - u32 flags, phy_interface_t iface); struct mii_bus *of_mdio_find_bus(struct device_node *mdio_np); int of_phy_register_fixed_link(struct device_node *np); @@ -100,13 +97,6 @@ of_phy_get_and_connect(struct net_device *dev, struct device_node *np, return NULL; } -static inline struct phy_device *of_phy_attach(struct net_device *dev, - struct device_node *phy_np, - u32 flags, phy_interface_t iface) -{ - return NULL; -} - static inline struct mii_bus *of_mdio_find_bus(struct device_node *mdio_np) { return NULL; -- cgit v1.2.3 From 20e07e2c3cf310578ef19fb4f1e64dc9832abd9d Mon Sep 17 00:00:00 2001 From: Wong Vee Khee Date: Wed, 17 Feb 2021 17:57:05 +0800 Subject: net: stmmac: Add PCI bus info to ethtool driver query output This patch populates the PCI bus info in the ethtool driver query data. Users will be able to view PCI bus info using 'ethtool -i '. Signed-off-by: Wong Vee Khee Signed-off-by: David S. Miller --- drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c | 1 + drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c | 4 ++++ include/linux/stmmac.h | 1 + 3 files changed, 6 insertions(+) (limited to 'include/linux') diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index 1c9c67b641a2..751dfdeec41c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -236,6 +236,7 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, int ret; int i; + plat->pdev = pdev; plat->phy_addr = -1; plat->clk_csr = 5; plat->has_gmac = 0; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index 9e54f953634b..c5642985ef95 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -268,6 +268,10 @@ static void stmmac_ethtool_getdrvinfo(struct net_device *dev, strlcpy(info->driver, MAC100_ETHTOOL_NAME, sizeof(info->driver)); + if (priv->plat->pdev) { + strlcpy(info->bus_info, pci_name(priv->plat->pdev), + sizeof(info->bus_info)); + } strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); } diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index 15ca6b4167cc..a302982de2d7 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -202,5 +202,6 @@ struct plat_stmmacenet_data { bool vlan_fail_q_en; u8 vlan_fail_q; unsigned int eee_usecs_rate; + struct pci_dev *pdev; }; #endif -- cgit v1.2.3