From 2777e73fc154e2e87233bdcc0e2402b33815198e Mon Sep 17 00:00:00 2001 From: Maramaina Naresh Date: Tue, 19 Dec 2023 18:07:05 +0530 Subject: scsi: ufs: core: Add CPU latency QoS support for UFS driver Register UFS driver to CPU latency PM QoS framework to improve UFS device random I/O performance. PM QoS initialization will insert new QoS request into the CPU latency QoS list with the maximum latency PM_QOS_DEFAULT_VALUE value. The UFS driver will vote for performance mode on scale up and power save mode for scale down. If clock scaling feature is not enabled then voting will be based on clock on or off condition. Also provide a sysfs interface to enable/disable PM QoS feature. tiotest benchmark tool I/O performance results on sm8550 platform: 1. Without PM QoS support Type (Speed in) | Average of 18 iterations Random Write(IPOS) | 41065.13 Random Read(IPOS) | 37101.3 2. With PM QoS support Type (Speed in) | Average of 18 iterations Random Write(IPOS) | 46784.9 Random Read(IPOS) | 42943.4 (Improvement with PM QoS = ~15%). Reviewed-by: Peter Wang Reviewed-by: AngeloGioacchino Del Regno Co-developed-by: Nitin Rawat Signed-off-by: Nitin Rawat Co-developed-by: Naveen Kumar Goud Arepalli Signed-off-by: Naveen Kumar Goud Arepalli Signed-off-by: Maramaina Naresh Link: https://lore.kernel.org/r/20231219123706.6463-2-quic_mnaresh@quicinc.com Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- include/ufs/ufshcd.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include') diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 8e2bce9a4f21..c491671e79b7 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -914,6 +914,8 @@ enum ufshcd_mcq_opr { * @dev_cmd_queue: Queue for issuing device management commands * @mcq_opr: MCQ operation and runtime registers * @ufs_rtc_update_work: A work for UFS RTC periodic update + * @pm_qos_req: PM QoS request handle + * @pm_qos_enabled: flag to check if pm qos is enabled */ struct ufs_hba { void __iomem *mmio_base; @@ -1080,6 +1082,8 @@ struct ufs_hba { struct ufshcd_mcq_opr_info_t mcq_opr[OPR_MAX]; struct delayed_work ufs_rtc_update_work; + struct pm_qos_request pm_qos_req; + bool pm_qos_enabled; }; /** @@ -1400,6 +1404,8 @@ int ufshcd_suspend_prepare(struct device *dev); int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm); void ufshcd_resume_complete(struct device *dev); bool ufshcd_is_hba_active(struct ufs_hba *hba); +void ufshcd_pm_qos_init(struct ufs_hba *hba); +void ufshcd_pm_qos_exit(struct ufs_hba *hba); /* Wrapper functions for safely calling variant operations */ static inline int ufshcd_vops_init(struct ufs_hba *hba) -- cgit v1.2.3 From 4380e64a94e16c757552e8e2fbdc856415012fc8 Mon Sep 17 00:00:00 2001 From: Peter Wang Date: Tue, 9 Jan 2024 20:40:14 +0800 Subject: scsi: core: Move autosuspend timer delay to Scsi_Host The runtime suspend timer delay is a const value in scsi_host_template which a host driver cannot modify at runtime. Move the delay to Scsi_Host to allow a driver to update it. Signed-off-by: Peter Wang Link: https://lore.kernel.org/r/20240109124015.31359-2-peter.wang@mediatek.com Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- drivers/scsi/sd.c | 2 +- drivers/ufs/core/ufshcd.c | 9 +++++++-- include/scsi/scsi_host.h | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) (limited to 'include') diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 0833b3e6aa6e..d1b87670764c 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -3728,7 +3728,7 @@ static int sd_probe(struct device *dev) blk_pm_runtime_init(sdp->request_queue, dev); if (sdp->rpm_autosuspend) { pm_runtime_set_autosuspend_delay(dev, - sdp->host->hostt->rpm_autosuspend_delay); + sdp->host->rpm_autosuspend_delay); } error = device_add_disk(dev, gd, NULL); diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 10b0c9cea0f0..e519695b704f 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -8031,11 +8031,13 @@ out: static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) { + struct Scsi_Host *shost = sdev->host; + scsi_autopm_get_device(sdev); blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev); if (sdev->rpm_autosuspend) pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev, - RPM_AUTOSUSPEND_DELAY_MS); + shost->rpm_autosuspend_delay); scsi_autopm_put_device(sdev); } @@ -9109,7 +9111,6 @@ static const struct scsi_host_template ufshcd_driver_template = { .track_queue_depth = 1, .skip_settle_delay = 1, .sdev_groups = ufshcd_driver_groups, - .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS, }; static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg, @@ -10568,6 +10569,10 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) host->max_cmd_len = UFS_CDB_SIZE; host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING); + /* Use default RPM delay if host not set */ + if (host->rpm_autosuspend_delay == 0) + host->rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS; + hba->max_pwr_info.is_valid = false; /* Initialize work queues */ diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index 3b907fc2ef08..b259d42a1e1a 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h @@ -497,9 +497,6 @@ struct scsi_host_template { * scsi_netlink.h */ u64 vendor_id; - - /* Delay for runtime autosuspend */ - int rpm_autosuspend_delay; }; /* @@ -713,6 +710,9 @@ struct Scsi_Host { */ struct device *dma_dev; + /* Delay for runtime autosuspend */ + int rpm_autosuspend_delay; + /* * We should ensure that this is aligned, both for better performance * and also because some compilers (m68k) don't automatically force -- cgit v1.2.3 From ab3e6c4e0ea149f16d5b719ecf7572862060d215 Mon Sep 17 00:00:00 2001 From: ChanWoo Lee Date: Tue, 2 Jan 2024 10:42:22 +0900 Subject: scsi: ufs: mcq: Add definition for REG_UFS_MEM_CFG register Instead of hardcoding the register field, add the proper definition. While at it, let's also use ufshcd_rmwl() to simplify updating this register. Reviewed-by: Peter Wang Signed-off-by: ChanWoo Lee Link: https://lore.kernel.org/r/20240102014222.23351-1-cw9316.lee@samsung.com Reviewed-by: Manivannan Sadhasivam Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufs-mcq.c | 6 ++++++ drivers/ufs/core/ufshcd.c | 4 +--- drivers/ufs/host/ufs-mediatek.c | 4 +--- include/ufs/ufshcd.h | 1 + include/ufs/ufshci.h | 3 +++ 5 files changed, 12 insertions(+), 6 deletions(-) (limited to 'include') diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index 0787456c2b89..edc752e55878 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -399,6 +399,12 @@ void ufshcd_mcq_enable_esi(struct ufs_hba *hba) } EXPORT_SYMBOL_GPL(ufshcd_mcq_enable_esi); +void ufshcd_mcq_enable(struct ufs_hba *hba) +{ + ufshcd_rmwl(hba, MCQ_MODE_SELECT, MCQ_MODE_SELECT, REG_UFS_MEM_CFG); +} +EXPORT_SYMBOL_GPL(ufshcd_mcq_enable); + void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg) { ufshcd_writel(hba, msg->address_lo, REG_UFS_ESILBA); diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index e519695b704f..f10a92681bba 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -8847,9 +8847,7 @@ static void ufshcd_config_mcq(struct ufs_hba *hba) hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED; - /* Select MCQ mode */ - ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x1, - REG_UFS_MEM_CFG); + ufshcd_mcq_enable(hba); hba->mcq_enabled = true; dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n", diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c index 72a7b3a7cc00..b8a8801322e2 100644 --- a/drivers/ufs/host/ufs-mediatek.c +++ b/drivers/ufs/host/ufs-mediatek.c @@ -1254,9 +1254,7 @@ static int ufs_mtk_link_set_hpm(struct ufs_hba *hba) ufs_mtk_config_mcq(hba, false); ufshcd_mcq_make_queues_operational(hba); ufshcd_mcq_config_mac(hba, hba->nutrs); - /* Enable MCQ mode */ - ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x1, - REG_UFS_MEM_CFG); + ufshcd_mcq_enable(hba); } return 0; diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index c491671e79b7..cb2afcebbdf5 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -1267,6 +1267,7 @@ unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba, struct ufs_hw_queue *hwq); void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba); void ufshcd_mcq_enable_esi(struct ufs_hba *hba); +void ufshcd_mcq_enable(struct ufs_hba *hba); void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg); int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table, diff --git a/include/ufs/ufshci.h b/include/ufs/ufshci.h index d5accacae6bc..a196e1c4c3bb 100644 --- a/include/ufs/ufshci.h +++ b/include/ufs/ufshci.h @@ -282,6 +282,9 @@ enum { /* UTMRLRSR - UTP Task Management Request Run-Stop Register 80h */ #define UTP_TASK_REQ_LIST_RUN_STOP_BIT 0x1 +/* REG_UFS_MEM_CFG - Global Config Registers 300h */ +#define MCQ_MODE_SELECT BIT(0) + /* CQISy - CQ y Interrupt Status Register */ #define UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS 0x1 -- cgit v1.2.3 From 994724e6b3f05fb3b6e4b1e87d7e074b65d47bf9 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Mon, 22 Jan 2024 18:22:02 -0600 Subject: scsi: core: Allow passthrough to request midlayer retries For passthrough we don't retry any error which we get a check condition for. This results in a lot of callers driving their own retries for all UAs, specific UAs, NOT_READY, specific sense values or any type of failure. This adds the core code to allow passthrough users to specify what errors they want the SCSI midlayer to retry for them. We can then convert users to drop a lot of their sense parsing and retry handling. Signed-off-by: Mike Christie Link: https://lore.kernel.org/r/20240123002220.129141-2-michael.christie@oracle.com Reviewed-by: John Garry Acked-by: Christoph Hellwig Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_lib.c | 98 ++++++++++++++++++++++++++++++++++++++++++++-- include/scsi/scsi_device.h | 48 +++++++++++++++++++++++ 2 files changed, 143 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index cf3864f72093..16a33a558842 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -184,6 +184,92 @@ void scsi_queue_insert(struct scsi_cmnd *cmd, int reason) __scsi_queue_insert(cmd, reason, true); } +void scsi_failures_reset_retries(struct scsi_failures *failures) +{ + struct scsi_failure *failure; + + failures->total_retries = 0; + + for (failure = failures->failure_definitions; failure->result; + failure++) + failure->retries = 0; +} +EXPORT_SYMBOL_GPL(scsi_failures_reset_retries); + +/** + * scsi_check_passthrough - Determine if passthrough scsi_cmnd needs a retry. + * @scmd: scsi_cmnd to check. + * @failures: scsi_failures struct that lists failures to check for. + * + * Returns -EAGAIN if the caller should retry else 0. + */ +static int scsi_check_passthrough(struct scsi_cmnd *scmd, + struct scsi_failures *failures) +{ + struct scsi_failure *failure; + struct scsi_sense_hdr sshdr; + enum sam_status status; + + if (!failures) + return 0; + + for (failure = failures->failure_definitions; failure->result; + failure++) { + if (failure->result == SCMD_FAILURE_RESULT_ANY) + goto maybe_retry; + + if (host_byte(scmd->result) && + host_byte(scmd->result) == host_byte(failure->result)) + goto maybe_retry; + + status = status_byte(scmd->result); + if (!status) + continue; + + if (failure->result == SCMD_FAILURE_STAT_ANY && + !scsi_status_is_good(scmd->result)) + goto maybe_retry; + + if (status != status_byte(failure->result)) + continue; + + if (status_byte(failure->result) != SAM_STAT_CHECK_CONDITION || + failure->sense == SCMD_FAILURE_SENSE_ANY) + goto maybe_retry; + + if (!scsi_command_normalize_sense(scmd, &sshdr)) + return 0; + + if (failure->sense != sshdr.sense_key) + continue; + + if (failure->asc == SCMD_FAILURE_ASC_ANY) + goto maybe_retry; + + if (failure->asc != sshdr.asc) + continue; + + if (failure->ascq == SCMD_FAILURE_ASCQ_ANY || + failure->ascq == sshdr.ascq) + goto maybe_retry; + } + + return 0; + +maybe_retry: + if (failure->allowed) { + if (failure->allowed == SCMD_FAILURE_NO_LIMIT || + ++failure->retries <= failure->allowed) + return -EAGAIN; + } else { + if (failures->total_allowed == SCMD_FAILURE_NO_LIMIT || + ++failures->total_retries <= failures->total_allowed) + return -EAGAIN; + } + + return 0; +} + /** * scsi_execute_cmd - insert request and wait for the result * @sdev: scsi_device @@ -192,7 +278,7 @@ void scsi_queue_insert(struct scsi_cmnd *cmd, int reason) * @buffer: data buffer * @bufflen: len of buffer * @timeout: request timeout in HZ - * @retries: number of times to retry request + * @ml_retries: number of times SCSI midlayer will retry request * @args: Optional args. See struct definition for field descriptions * * Returns the scsi_cmnd result field if a command was executed, or a negative @@ -200,7 +286,7 @@ void scsi_queue_insert(struct scsi_cmnd *cmd, int reason) */ int scsi_execute_cmd(struct scsi_device *sdev, const unsigned char *cmd, blk_opf_t opf, void *buffer, unsigned int bufflen, - int timeout, int retries, + int timeout, int ml_retries, const struct scsi_exec_args *args) { static const struct scsi_exec_args default_args; @@ -214,6 +300,7 @@ int scsi_execute_cmd(struct scsi_device *sdev, const unsigned char *cmd, args->sense_len != SCSI_SENSE_BUFFERSIZE)) return -EINVAL; +retry: req = scsi_alloc_request(sdev->request_queue, opf, args->req_flags); if (IS_ERR(req)) return PTR_ERR(req); @@ -227,7 +314,7 @@ int scsi_execute_cmd(struct scsi_device *sdev, const unsigned char *cmd, scmd = blk_mq_rq_to_pdu(req); scmd->cmd_len = COMMAND_SIZE(cmd[0]); memcpy(scmd->cmnd, cmd, scmd->cmd_len); - scmd->allowed = retries; + scmd->allowed = ml_retries; scmd->flags |= args->scmd_flags; req->timeout = timeout; req->rq_flags |= RQF_QUIET; @@ -237,6 +324,11 @@ int scsi_execute_cmd(struct scsi_device *sdev, const unsigned char *cmd, */ blk_execute_rq(req, true); + if (scsi_check_passthrough(scmd, args->failures) == -EAGAIN) { + blk_mq_free_request(req); + goto retry; + } + /* * Some devices (USB mass-storage in particular) may transfer * garbage data together with a residue indicating that the data diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h index 5ec1e71a09de..4dceabb9dbe1 100644 --- a/include/scsi/scsi_device.h +++ b/include/scsi/scsi_device.h @@ -489,6 +489,52 @@ extern int scsi_is_sdev_device(const struct device *); extern int scsi_is_target_device(const struct device *); extern void scsi_sanitize_inquiry_string(unsigned char *s, int len); +/* + * scsi_execute_cmd users can set scsi_failure.result to have + * scsi_check_passthrough fail/retry a command. scsi_failure.result can be a + * specific host byte or message code, or SCMD_FAILURE_RESULT_ANY can be used + * to match any host or message code. + */ +#define SCMD_FAILURE_RESULT_ANY 0x7fffffff +/* + * Set scsi_failure.result to SCMD_FAILURE_STAT_ANY to fail/retry any failure + * scsi_status_is_good returns false for. + */ +#define SCMD_FAILURE_STAT_ANY 0xff +/* + * The following can be set to the scsi_failure sense, asc and ascq fields to + * match on any sense, ASC, or ASCQ value. + */ +#define SCMD_FAILURE_SENSE_ANY 0xff +#define SCMD_FAILURE_ASC_ANY 0xff +#define SCMD_FAILURE_ASCQ_ANY 0xff +/* Always retry a matching failure. */ +#define SCMD_FAILURE_NO_LIMIT -1 + +struct scsi_failure { + int result; + u8 sense; + u8 asc; + u8 ascq; + /* + * Number of times scsi_execute_cmd will retry the failure. It does + * not count for the total_allowed. + */ + s8 allowed; + /* Number of times the failure has been retried. */ + s8 retries; +}; + +struct scsi_failures { + /* + * If a scsi_failure does not have a retry limit setup this limit will + * be used. + */ + int total_allowed; + int total_retries; + struct scsi_failure *failure_definitions; +}; + /* Optional arguments to scsi_execute_cmd */ struct scsi_exec_args { unsigned char *sense; /* sense buffer */ @@ -497,12 +543,14 @@ struct scsi_exec_args { blk_mq_req_flags_t req_flags; /* BLK_MQ_REQ flags */ int scmd_flags; /* SCMD flags */ int *resid; /* residual length */ + struct scsi_failures *failures; /* failures to retry */ }; int scsi_execute_cmd(struct scsi_device *sdev, const unsigned char *cmd, blk_opf_t opf, void *buffer, unsigned int bufflen, int timeout, int retries, const struct scsi_exec_args *args); +void scsi_failures_reset_retries(struct scsi_failures *failures); extern void sdev_disable_disk_events(struct scsi_device *sdev); extern void sdev_enable_disk_events(struct scsi_device *sdev); -- cgit v1.2.3