diff --git a/Documentation/driver-api/cxl/linux/dax-driver.rst b/Documentation/driver-api/cxl/linux/dax-driver.rst index 10d953a2167b..72c85a0f8606 100644 --- a/Documentation/driver-api/cxl/linux/dax-driver.rst +++ b/Documentation/driver-api/cxl/linux/dax-driver.rst @@ -35,9 +35,9 @@ will be exposed to the kernel page allocator in the user-selected memory zone. The :code:`memmap_on_memory` setting (both global and DAX device local) -dictates where the kernell will allocate the :code:`struct folio` descriptors +dictates where the kernel will allocate the :code:`struct folio` descriptors for this memory will come from. If :code:`memmap_on_memory` is set, memory hotplug will set aside a portion of the memory block capacity to allocate folios. If unset, the memory is allocated via a normal :code:`GFP_KERNEL` -allocation - and as a result will most likely land on the local NUM node of the +allocation - and as a result will most likely land on the local NUMA node of the CPU executing the hotplug operation. diff --git a/drivers/cxl/core/features.c b/drivers/cxl/core/features.c index 85185af46b72..ba6d2a5acb74 100644 --- a/drivers/cxl/core/features.c +++ b/drivers/cxl/core/features.c @@ -225,7 +225,7 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, void *feat_out, size_t feat_out_size, u16 offset, u16 *return_code) { - size_t data_to_rd_size, size_out; + size_t data_to_rd_size; struct cxl_mbox_get_feat_in pi; struct cxl_mbox_cmd mbox_cmd; size_t data_rcvd_size = 0; @@ -237,9 +237,10 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, if (!feat_out || !feat_out_size) return 0; - size_out = min(feat_out_size, cxl_mbox->payload_size); uuid_copy(&pi.uuid, feat_uuid); pi.selection = selection; + + guard(mutex)(&cxl_mbox->feat_mutex); do { data_to_rd_size = min(feat_out_size - data_rcvd_size, cxl_mbox->payload_size); @@ -250,7 +251,7 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, .opcode = CXL_MBOX_OP_GET_FEATURE, .size_in = sizeof(pi), .payload_in = &pi, - .size_out = size_out, + .size_out = data_to_rd_size, .payload_out = feat_out + data_rcvd_size, .min_out = data_to_rd_size, }; @@ -314,6 +315,7 @@ int cxl_set_feature(struct cxl_mailbox *cxl_mbox, data_in_size = cxl_mbox->payload_size - hdr_size; } + guard(mutex)(&cxl_mbox->feat_mutex); do { int rc; @@ -471,6 +473,10 @@ static void *cxlctl_get_feature(struct cxl_features_state *cxlfs, if (!count) return ERR_PTR(-EINVAL); + if (out_size < offsetof(struct fwctl_rpc_cxl_out, payload) || + count > out_size - offsetof(struct fwctl_rpc_cxl_out, payload)) + return ERR_PTR(-EINVAL); + struct fwctl_rpc_cxl_out *rpc_out __free(kvfree) = kvzalloc(out_size, GFP_KERNEL); if (!rpc_out) @@ -516,6 +522,9 @@ static void *cxlctl_set_feature(struct cxl_features_state *cxlfs, flags = le32_to_cpu(feat_in->flags); out_size = *out_len; + if (out_size < offsetof(struct fwctl_rpc_cxl_out, payload)) + return ERR_PTR(-EINVAL); + struct fwctl_rpc_cxl_out *rpc_out __free(kvfree) = kvzalloc(out_size, GFP_KERNEL); if (!rpc_out) @@ -649,7 +658,13 @@ static void *cxlctl_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope, struct cxl_memdev *cxlmd = fwctl_to_memdev(fwctl_dev); struct cxl_features_state *cxlfs = to_cxlfs(cxlmd->cxlds); const struct fwctl_rpc_cxl *rpc_in = in; - u16 opcode = rpc_in->opcode; + u16 opcode; + + if (in_len < sizeof(rpc_in->hdr) || + rpc_in->op_size > in_len - sizeof(rpc_in->hdr)) + return ERR_PTR(-EINVAL); + + opcode = rpc_in->opcode; if (!cxlctl_validate_hw_command(cxlfs, rpc_in, scope, opcode)) return ERR_PTR(-EINVAL); diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 49949869fac3..55828a836c01 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -11,7 +11,6 @@ #include "core.h" #include "trace.h" -#include "mce.h" static bool cxl_raw_allow_all; @@ -387,11 +386,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox_cmd, } } - /* Prepare to handle a full payload for variable sized output */ - if (out_size == CXL_VARIABLE_PAYLOAD) - mbox_cmd->size_out = cxl_mbox->payload_size; - else - mbox_cmd->size_out = out_size; + mbox_cmd->size_out = min_t(size_t, out_size, cxl_mbox->payload_size); if (mbox_cmd->size_out) { mbox_cmd->payload_out = kvzalloc(mbox_cmd->size_out, GFP_KERNEL); @@ -1413,6 +1408,11 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, if (rc) break; + if (!le16_to_cpu(po->count)) { + dev_dbg(&cxlmd->dev, "Poison empty payload!\n"); + break; + } + for (int i = 0; i < le16_to_cpu(po->count); i++) trace_cxl_poison(cxlmd, cxlr, &po->record[i], po->flags, po->overflow_ts, @@ -1474,6 +1474,7 @@ int cxl_mailbox_init(struct cxl_mailbox *cxl_mbox, struct device *host) cxl_mbox->host = host; mutex_init(&cxl_mbox->mbox_mutex); + mutex_init(&cxl_mbox->feat_mutex); rcuwait_init(&cxl_mbox->mbox_wait); return 0; @@ -1484,7 +1485,6 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev, u64 serial, u16 dvsec) { struct cxl_memdev_state *mds; - int rc; mds = devm_cxl_dev_state_create(dev, CXL_DEVTYPE_CLASSMEM, serial, dvsec, struct cxl_memdev_state, cxlds, @@ -1496,12 +1496,6 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev, u64 serial, mutex_init(&mds->event.log_lock); - rc = devm_cxl_register_mce_notifier(dev, &mds->mce_notifier); - if (rc == -EOPNOTSUPP) - dev_warn(dev, "CXL MCE unsupported\n"); - else if (rc) - return ERR_PTR(rc); - return mds; } EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, "CXL"); diff --git a/drivers/cxl/core/mce.c b/drivers/cxl/core/mce.c index ff8d078c6ca1..65fed913b221 100644 --- a/drivers/cxl/core/mce.c +++ b/drivers/cxl/core/mce.c @@ -4,16 +4,16 @@ #include #include #include -#include +#include +#include "core.h" #include "mce.h" static int cxl_handle_mce(struct notifier_block *nb, unsigned long val, void *data) { - struct cxl_memdev_state *mds = container_of(nb, struct cxl_memdev_state, - mce_notifier); - struct cxl_memdev *cxlmd = mds->cxlds.cxlmd; - struct cxl_port *endpoint = cxlmd->endpoint; + struct cxl_region *cxlr = container_of(nb, struct cxl_region, + mce_notifier); + struct cxl_region_params *p = &cxlr->params; struct mce *mce = data; u64 spa, spa_alias; unsigned long pfn; @@ -21,26 +21,25 @@ static int cxl_handle_mce(struct notifier_block *nb, unsigned long val, if (!mce || !mce_usable_address(mce)) return NOTIFY_DONE; - if (!endpoint) - return NOTIFY_DONE; - spa = mce->addr & MCI_ADDR_PHYSADDR; - pfn = spa >> PAGE_SHIFT; - if (!pfn_valid(pfn)) + if (!cxl_resource_contains_addr(p->res, spa)) return NOTIFY_DONE; - spa_alias = cxl_port_get_spa_cache_alias(endpoint, spa); - if (spa_alias == ~0ULL) - return NOTIFY_DONE; + if (spa >= p->res->start + p->cache_size) + spa_alias = spa - p->cache_size; + else + spa_alias = spa + p->cache_size; pfn = spa_alias >> PAGE_SHIFT; + if (!pfn_valid(pfn)) + return NOTIFY_DONE; /* * Take down the aliased memory page. The original memory page flagged * by the MCE will be taken cared of by the standard MCE handler. */ - dev_emerg(mds->cxlds.dev, "Offlining aliased SPA address0: %#llx\n", + dev_emerg(&cxlr->dev, "Offlining aliased SPA address0: %#llx\n", spa_alias); if (!memory_failure(pfn, 0)) set_mce_nospec(pfn); diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index 2e457b1ebc7d..b3419df586b9 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -988,7 +988,7 @@ static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, if (!size) return FW_UPLOAD_ERR_INVALID_SIZE; - mds->fw.oneshot = struct_size(transfer, data, size) < + mds->fw.oneshot = struct_size(transfer, data, size) <= cxl_mbox->payload_size; if (cxl_mem_get_fw_info(mds)) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index cb633e19151b..312210198435 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -1750,8 +1750,8 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, parent_dport, uport_dev, dport_dev); if (IS_ERR(dport)) { - /* Port already exists, restart iteration */ - if (PTR_ERR(dport) == -EAGAIN) + /* Port or dport already exists, restart iteration */ + if (PTR_ERR(dport) == -EAGAIN || PTR_ERR(dport) == -EBUSY) return 0; return PTR_ERR(dport); } diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 1e211542b6b6..578622240401 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -15,6 +15,7 @@ #include #include #include "core.h" +#include "mce.h" /** * DOC: cxl core region @@ -3859,34 +3860,6 @@ int cxl_add_to_region(struct cxl_endpoint_decoder *cxled) } EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL"); -u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa) -{ - struct cxl_region_ref *iter; - unsigned long index; - - if (!endpoint) - return ~0ULL; - - guard(rwsem_write)(&cxl_rwsem.region); - - xa_for_each(&endpoint->regions, index, iter) { - struct cxl_region_params *p = &iter->region->params; - - if (cxl_resource_contains_addr(p->res, spa)) { - if (!p->cache_size) - return ~0ULL; - - if (spa >= p->res->start + p->cache_size) - return spa - p->cache_size; - - return spa + p->cache_size; - } - } - - return ~0ULL; -} -EXPORT_SYMBOL_NS_GPL(cxl_port_get_spa_cache_alias, "CXL"); - static int is_system_ram(struct resource *res, void *arg) { struct cxl_region *cxlr = arg; @@ -4217,6 +4190,20 @@ static int cxl_region_probe(struct device *dev) if (rc) return rc; + /* + * Regions fronted by an extended linear cache need the MCE notifier to + * offline the aliased page on a memory error. + */ + if (p->cache_size) { + rc = devm_cxl_register_mce_notifier(&cxlr->dev, + &cxlr->mce_notifier); + if (rc == -EOPNOTSUPP) + dev_warn(&cxlr->dev, + "CONFIG_CXL_MCE disabled, MCE notifier not registered\n"); + else if (rc) + return rc; + } + rc = cxl_region_setup_poison(cxlr); if (rc) return rc; diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index c0e5308e4d1b..bdd0132c999d 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -478,6 +478,7 @@ struct cxl_region_params { * @coord: QoS access coordinates for the region * @node_notifier: notifier for setting the access coordinates to node * @adist_notifier: notifier for calculating the abstract distance of node + * @mce_notifier: notifier for MCE */ struct cxl_region { struct device dev; @@ -493,6 +494,7 @@ struct cxl_region { struct access_coordinate coord[ACCESS_COORDINATE_MAX]; struct notifier_block node_notifier; struct notifier_block adist_notifier; + struct notifier_block mce_notifier; }; struct cxl_nvdimm_bridge { @@ -870,7 +872,6 @@ bool is_cxl_pmem_region(struct device *dev); struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev); int cxl_add_to_region(struct cxl_endpoint_decoder *cxled); struct cxl_dax_region *to_cxl_dax_region(struct device *dev); -u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa); bool cxl_region_contains_resource(const struct resource *res); #else static inline bool is_cxl_pmem_region(struct device *dev) @@ -889,11 +890,6 @@ static inline struct cxl_dax_region *to_cxl_dax_region(struct device *dev) { return NULL; } -static inline u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, - u64 spa) -{ - return 0; -} static inline bool cxl_region_contains_resource(const struct resource *res) { return false; diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index ed419d0c59f2..c401e3a1af06 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -183,7 +183,7 @@ static inline struct cxl_ep *cxl_ep_load(struct cxl_port *port, C(MBUNSUPPORTED, -ENXIO, "unsupported on the mailbox it was issued on"),\ C(PAYLOADLEN, -ENXIO, "invalid payload length"), \ C(LOG, -ENXIO, "invalid or unsupported log page"), \ - C(INTERRUPTED, -ENXIO, "asynchronous event occured"), \ + C(INTERRUPTED, -ENXIO, "asynchronous event occurred"), \ C(FEATUREVERSION, -ENXIO, "unsupported feature version"), \ C(FEATURESELVALUE, -ENXIO, "unsupported feature selection value"), \ C(FEATURETRANSFERIP, -ENXIO, "feature transfer in progress"), \ @@ -431,7 +431,6 @@ static inline struct cxl_dev_state *mbox_to_cxlds(struct cxl_mailbox *cxl_mbox) * @poison: poison driver state info * @security: security driver state info * @fw: firmware upload / activation state - * @mce_notifier: MCE notifier * * See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for * details on capacity parameters. @@ -451,7 +450,6 @@ struct cxl_memdev_state { struct cxl_poison_state poison; struct cxl_security_state security; struct cxl_fw_state fw; - struct notifier_block mce_notifier; }; static inline struct cxl_memdev_state * diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index bb892dbfdd6d..d7f499cf1d8f 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -692,12 +692,6 @@ static int cxl_pci_type3_init_mailbox(struct cxl_dev_state *cxlds) { int rc; - /* - * Fail the init if there's no mailbox. For a type3 this is out of spec. - */ - if (!cxlds->reg_map.device_map.mbox.valid) - return -ENODEV; - rc = cxl_mailbox_init(&cxlds->cxl_mbox, cxlds->dev); if (rc) return rc; @@ -830,10 +824,13 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) */ rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &cxlds->reg_map); - if (rc) + if (rc) { + if (rc == -EPROBE_DEFER) + return rc; dev_warn(&pdev->dev, "No component registers (%d)\n", rc); - else if (!cxlds->reg_map.component_map.ras.valid) + } else if (!cxlds->reg_map.component_map.ras.valid) { dev_dbg(&pdev->dev, "RAS registers not found\n"); + } rc = cxl_pci_type3_init_mailbox(cxlds); if (rc) diff --git a/include/cxl/mailbox.h b/include/cxl/mailbox.h index c4e99e2e3a9d..d008b9db07aa 100644 --- a/include/cxl/mailbox.h +++ b/include/cxl/mailbox.h @@ -50,6 +50,7 @@ struct cxl_mbox_cmd { * @payload_size: Size of space for payload * (CXL 3.1 8.2.8.4.3 Mailbox Capabilities Register) * @mbox_mutex: mutex protects device mailbox and firmware + * @feat_mutex: serializes multi-part Get/Set Feature transfers * @mbox_wait: rcuwait for mailbox * @mbox_send: @dev specific transport for transmitting mailbox commands * @feat_cap: Features capability @@ -60,6 +61,7 @@ struct cxl_mailbox { DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); size_t payload_size; struct mutex mbox_mutex; /* lock to protect mailbox context */ + struct mutex feat_mutex; struct rcuwait mbox_wait; int (*mbox_send)(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd); enum cxl_features_capability feat_cap;