From 11270e526276ffad4c4237acb393da82a3287487 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:21 -0700 Subject: base/node / ACPI: Enumerate node access class for 'struct access_coordinate' Both generic node and HMAT handling code have been using magic numbers to indicate access classes for 'struct access_coordinate'. Introduce enums to enumerate the access0 and access1 classes shared by the two subsystems. Update the function parameters and callers as appropriate to utilize the new enum. Access0 is named to ACCESS_COORDINATE_LOCAL in order to indicate that the access class is for 'struct access_coordinate' between a target node and the nearest initiator node. Access1 is named to ACCESS_COORDINATE_CPU in order to indicate that the access class is for 'struct access_coordinate' between a target node and the nearest CPU node. Cc: Greg Kroah-Hartman Cc: Rafael J. Wysocki Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Acked-by: Greg Kroah-Hartman Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-3-dave.jiang@intel.com Signed-off-by: Dan Williams --- include/linux/node.h | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/node.h b/include/linux/node.h index 25b66d705ee2..dfc004e4bee7 100644 --- a/include/linux/node.h +++ b/include/linux/node.h @@ -34,6 +34,18 @@ struct access_coordinate { unsigned int write_latency; }; +/* + * ACCESS_COORDINATE_LOCAL correlates to ACCESS CLASS 0 + * - access_coordinate between target node and nearest initiator node + * ACCESS_COORDINATE_CPU correlates to ACCESS CLASS 1 + * - access_coordinate between target node and nearest CPU node + */ +enum access_coordinate_class { + ACCESS_COORDINATE_LOCAL, + ACCESS_COORDINATE_CPU, + ACCESS_COORDINATE_MAX +}; + enum cache_indexing { NODE_CACHE_DIRECT_MAP, NODE_CACHE_INDEXED, @@ -66,7 +78,7 @@ struct node_cache_attrs { #ifdef CONFIG_HMEM_REPORTING void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs); void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, - unsigned access); + enum access_coordinate_class access); #else static inline void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs) @@ -75,7 +87,7 @@ static inline void node_add_cache(unsigned int nid, static inline void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, - unsigned access) + enum access_coordinate_class access) { } #endif @@ -137,7 +149,7 @@ extern void unregister_memory_block_under_nodes(struct memory_block *mem_blk); extern int register_memory_node_under_compute_node(unsigned int mem_nid, unsigned int cpu_nid, - unsigned access); + enum access_coordinate_class access); #else static inline void node_dev_init(void) { -- cgit v1.2.3 From 067353a46d8ccdac279ebab97c038c3658e97541 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:30 -0700 Subject: cxl/region: Add memory hotplug notifier for cxl region When the CXL region is formed, the driver computes the performance data for the region. However this data is not available at the node data collection that has been populated by the HMAT during kernel initialization. Add a memory hotplug notifier to update the access coordinates to the 'struct memory_target' context kept by the HMAT_REPORTING code. Add CXL_CALLBACK_PRI for a memory hotplug callback priority. Set the priority number to be called before HMAT_CALLBACK_PRI. The CXL update must happen before hmat_callback(). A new HMAT_REPORTING helper hmat_update_target_coordinates() is added in order to allow CXL to update the memory_target access coordinates. A new ext_updated member is added to the memory_target to indicate that the access coordinates within the memory_target has been updated by an external agent such as CXL. This prevents data being overwritten by the hmat_update_target_attrs() triggered by hmat_callback(). Cc: Andrew Morton Cc: Rafael J. Wysocki Reviewed-by: Huang, Ying Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-12-dave.jiang@intel.com Signed-off-by: Dan Williams --- drivers/acpi/numa/hmat.c | 34 ++++++++++++++++++++++++ drivers/cxl/core/cdat.c | 6 +++++ drivers/cxl/core/core.h | 3 +++ drivers/cxl/core/region.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++ drivers/cxl/cxl.h | 3 +++ include/linux/acpi.h | 12 +++++++++ include/linux/memory.h | 1 + 7 files changed, 127 insertions(+) (limited to 'include/linux') diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index 75e9aac43228..2c8ccc91ebe6 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -74,6 +74,7 @@ struct memory_target { struct node_cache_attrs cache_attrs; u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE]; bool registered; + bool ext_updated; /* externally updated */ }; struct memory_initiator { @@ -328,6 +329,35 @@ static void hmat_update_target_access(struct memory_target *target, } } +int hmat_update_target_coordinates(int nid, struct access_coordinate *coord, + enum access_coordinate_class access) +{ + struct memory_target *target; + int pxm; + + if (nid == NUMA_NO_NODE) + return -EINVAL; + + pxm = node_to_pxm(nid); + guard(mutex)(&target_lock); + target = find_mem_target(pxm); + if (!target) + return -ENODEV; + + hmat_update_target_access(target, ACPI_HMAT_READ_LATENCY, + coord->read_latency, access); + hmat_update_target_access(target, ACPI_HMAT_WRITE_LATENCY, + coord->write_latency, access); + hmat_update_target_access(target, ACPI_HMAT_READ_BANDWIDTH, + coord->read_bandwidth, access); + hmat_update_target_access(target, ACPI_HMAT_WRITE_BANDWIDTH, + coord->write_bandwidth, access); + target->ext_updated = true; + + return 0; +} +EXPORT_SYMBOL_GPL(hmat_update_target_coordinates); + static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc) { struct memory_locality *loc; @@ -699,6 +729,10 @@ static void hmat_update_target_attrs(struct memory_target *target, u32 best = 0; int i; + /* Don't update if an external agent has changed the data. */ + if (target->ext_updated) + return; + /* Don't update for generic port if there's no device handle */ if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL || access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) && diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 40052666ebf1..ee1bc8fa396b 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -580,3 +580,9 @@ void cxl_region_perf_data_calculate(struct cxl_region *cxlr, DIV_ROUND_UP(cxlr->coord[i].write_latency, 1000); } } + +int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, + enum access_coordinate_class access) +{ + return hmat_update_target_coordinates(nid, &cxlr->coord[access], access); +} diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 3b64fb1b9ed0..e19800a7ce06 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -90,4 +90,7 @@ enum cxl_poison_trace_type { long cxl_pci_get_latency(struct pci_dev *pdev); +int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, + enum access_coordinate_class access); + #endif /* __CXL_CORE_H__ */ diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index d4d39536e6c7..535492ec8529 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -116,12 +117,22 @@ static const struct attribute_group cxl_region_access0_coordinate_group = { .is_visible = cxl_region_access0_coordinate_visible, }; +static const struct attribute_group *get_cxl_region_access0_group(void) +{ + return &cxl_region_access0_coordinate_group; +} + static const struct attribute_group cxl_region_access1_coordinate_group = { .name = "access1", .attrs = access1_coordinate_attrs, .is_visible = cxl_region_access1_coordinate_visible, }; +static const struct attribute_group *get_cxl_region_access1_group(void) +{ + return &cxl_region_access1_coordinate_group; +} + static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2216,6 +2227,7 @@ static void unregister_region(void *_cxlr) struct cxl_region_params *p = &cxlr->params; int i; + unregister_memory_notifier(&cxlr->memory_notifier); device_del(&cxlr->dev); /* @@ -2260,6 +2272,58 @@ static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int i return cxlr; } +static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) +{ + int cset = 0; + int rc; + + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { + if (cxlr->coord[i].read_bandwidth) { + rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); + if (rc == 0) + cset++; + } + } + + if (!cset) + return false; + + rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group()); + if (rc) + dev_dbg(&cxlr->dev, "Failed to update access0 group\n"); + + rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group()); + if (rc) + dev_dbg(&cxlr->dev, "Failed to update access1 group\n"); + + return true; +} + +static int cxl_region_perf_attrs_callback(struct notifier_block *nb, + unsigned long action, void *arg) +{ + struct cxl_region *cxlr = container_of(nb, struct cxl_region, + memory_notifier); + struct cxl_region_params *p = &cxlr->params; + struct cxl_endpoint_decoder *cxled = p->targets[0]; + struct cxl_decoder *cxld = &cxled->cxld; + struct memory_notify *mnb = arg; + int nid = mnb->status_change_nid; + int region_nid; + + if (nid == NUMA_NO_NODE || action != MEM_ONLINE) + return NOTIFY_DONE; + + region_nid = phys_to_target_node(cxld->hpa_range.start); + if (nid != region_nid) + return NOTIFY_DONE; + + if (!cxl_region_update_coordinates(cxlr, nid)) + return NOTIFY_DONE; + + return NOTIFY_OK; +} + /** * devm_cxl_add_region - Adds a region to a decoder * @cxlrd: root decoder @@ -2307,6 +2371,10 @@ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, if (rc) goto err; + cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback; + cxlr->memory_notifier.priority = CXL_CALLBACK_PRI; + register_memory_notifier(&cxlr->memory_notifier); + rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); if (rc) return ERR_PTR(rc); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 95864ce7b394..534e25e2f0a4 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -518,6 +519,7 @@ struct cxl_region_params { * @flags: Region state flags * @params: active + config params for the region * @coord: QoS access coordinates for the region + * @memory_notifier: notifier for setting the access coordinates to node */ struct cxl_region { struct device dev; @@ -529,6 +531,7 @@ struct cxl_region { unsigned long flags; struct cxl_region_params params; struct access_coordinate coord[ACCESS_COORDINATE_MAX]; + struct notifier_block memory_notifier; }; struct cxl_nvdimm_bridge { diff --git a/include/linux/acpi.h b/include/linux/acpi.h index b7165e52b3c6..c84c2f34b8ee 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1547,4 +1547,16 @@ static inline void acpi_use_parent_companion(struct device *dev) ACPI_COMPANION_SET(dev, ACPI_COMPANION(dev->parent)); } +#ifdef CONFIG_ACPI_HMAT +int hmat_update_target_coordinates(int nid, struct access_coordinate *coord, + enum access_coordinate_class access); +#else +static inline int hmat_update_target_coordinates(int nid, + struct access_coordinate *coord, + enum access_coordinate_class access) +{ + return -EOPNOTSUPP; +} +#endif + #endif /*_LINUX_ACPI_H*/ diff --git a/include/linux/memory.h b/include/linux/memory.h index f53cfdaaaa41..d8588256578a 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -114,6 +114,7 @@ struct mem_section; #define DEFAULT_CALLBACK_PRI 0 #define SLAB_CALLBACK_PRI 1 #define HMAT_CALLBACK_PRI 2 +#define CXL_CALLBACK_PRI 5 #define MM_COMPUTE_BATCH_PRI 10 #define CPUSET_CALLBACK_PRI 10 #define MEMTIER_HOTPLUG_PRI 100 -- cgit v1.2.3 From debdce20c4f28b7e5aa48512e7abf270a00e9051 Mon Sep 17 00:00:00 2001 From: Dave Jiang Date: Fri, 8 Mar 2024 14:59:31 -0700 Subject: cxl/region: Deal with numa nodes not enumerated by SRAT For the numa nodes that are not created by SRAT, no memory_target is allocated and is not managed by the HMAT_REPORTING code. Therefore hmat_callback() memory hotplug notifier will exit early on those NUMA nodes. The CXL memory hotplug notifier will need to call node_set_perf_attrs() directly in order to setup the access sysfs attributes. In acpi_numa_init(), the last proximity domain (pxm) id created by SRAT is stored. Add a helper function acpi_node_backed_by_real_pxm() in order to check if a NUMA node id is defined by SRAT or created by CFMWS. node_set_perf_attrs() symbol is exported to allow update of perf attribs for a node. The sysfs path of /sys/devices/system/node/nodeX/access0/initiators/* is created by node_set_perf_attrs() for the various attributes where nodeX is matched to the NUMA node of the CXL region. Cc: Rafael J. Wysocki Reviewed-by: Alison Schofield Reviewed-by: Jonathan Cameron Tested-by: Jonathan Cameron Signed-off-by: Dave Jiang Link: https://lore.kernel.org/r/20240308220055.2172956-13-dave.jiang@intel.com Signed-off-by: Dan Williams --- drivers/acpi/numa/srat.c | 11 +++++++++++ drivers/base/node.c | 1 + drivers/cxl/core/cdat.c | 5 +++++ drivers/cxl/core/core.h | 1 + drivers/cxl/core/region.c | 7 ++++++- include/linux/acpi.h | 9 +++++++++ 6 files changed, 33 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index 0214518fc582..e45e64993c50 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -29,6 +29,8 @@ static int node_to_pxm_map[MAX_NUMNODES] unsigned char acpi_srat_revision __initdata; static int acpi_numa __initdata; +static int last_real_pxm; + void __init disable_srat(void) { acpi_numa = -1; @@ -536,6 +538,7 @@ int __init acpi_numa_init(void) if (node_to_pxm_map[i] > fake_pxm) fake_pxm = node_to_pxm_map[i]; } + last_real_pxm = fake_pxm; fake_pxm++; acpi_table_parse_cedt(ACPI_CEDT_TYPE_CFMWS, acpi_parse_cfmws, &fake_pxm); @@ -547,6 +550,14 @@ int __init acpi_numa_init(void) return 0; } +bool acpi_node_backed_by_real_pxm(int nid) +{ + int pxm = node_to_pxm(nid); + + return pxm <= last_real_pxm; +} +EXPORT_SYMBOL_GPL(acpi_node_backed_by_real_pxm); + static int acpi_get_pxm(acpi_handle h) { unsigned long long pxm; diff --git a/drivers/base/node.c b/drivers/base/node.c index a73b0c9a401a..eb72580288e6 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -215,6 +215,7 @@ void node_set_perf_attrs(unsigned int nid, struct access_coordinate *coord, } } } +EXPORT_SYMBOL_GPL(node_set_perf_attrs); /** * struct node_cache_info - Internal tracking for memory node caches diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index ee1bc8fa396b..10d4f1d7dad1 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -586,3 +586,8 @@ int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, { return hmat_update_target_coordinates(nid, &cxlr->coord[access], access); } + +bool cxl_need_node_perf_attrs_update(int nid) +{ + return !acpi_node_backed_by_real_pxm(nid); +} diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index e19800a7ce06..bc5a95665aa0 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -92,5 +92,6 @@ long cxl_pci_get_latency(struct pci_dev *pdev); int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, enum access_coordinate_class access); +bool cxl_need_node_perf_attrs_update(int nid); #endif /* __CXL_CORE_H__ */ diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 535492ec8529..5c186e0a39b9 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -2279,7 +2279,12 @@ static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { if (cxlr->coord[i].read_bandwidth) { - rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); + rc = 0; + if (cxl_need_node_perf_attrs_update(nid)) + node_set_perf_attrs(nid, &cxlr->coord[i], i); + else + rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); + if (rc == 0) cset++; } diff --git a/include/linux/acpi.h b/include/linux/acpi.h index c84c2f34b8ee..2a7c4b90d589 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -1559,4 +1559,13 @@ static inline int hmat_update_target_coordinates(int nid, } #endif +#ifdef CONFIG_ACPI_NUMA +bool acpi_node_backed_by_real_pxm(int nid); +#else +static inline bool acpi_node_backed_by_real_pxm(int nid) +{ + return false; +} +#endif + #endif /*_LINUX_ACPI_H*/ -- cgit v1.2.3