45540b9fab
Changes:
- Rebase onto v4.19.206
Links:
- kernel: 9c073cfc7c
335 lines
10 KiB
Diff
335 lines
10 KiB
Diff
From 393a9f9995f490d4b17ef4713aa370111b67efeb Mon Sep 17 00:00:00 2001
|
|
From: kitakar5525 <34676735+kitakar5525@users.noreply.github.com>
|
|
Date: Sat, 28 Sep 2019 17:48:21 +0200
|
|
Subject: [PATCH] nvme: Backport changes for suspend
|
|
|
|
Backported commits are:
|
|
|
|
- torvalds/linux@4eaefe8c621c6195c91044396ed8060c179f7aae
|
|
(nvme-pci: Allow PCI bus-level PM to be used if ASPM is disabled)
|
|
|
|
- torvalds/linux@accd2dd72c8f087441d725dd916688171519e4e6
|
|
(PCI/ASPM: Add pcie_aspm_enabled())
|
|
|
|
- torvalds/linux@d916b1be94b6dc8d293abed2451f3062f6af7551
|
|
(nvme-pci: use host managed power state for suspend)
|
|
|
|
- torvalds/linux@1a87ee657c530bb2f3e39e4ac184d48f5f959cda
|
|
(nvme: export get and set features)
|
|
|
|
- torvalds/linux@d6135c3a1ec0cddda7b8b8e1b5b4abeeafd98289
|
|
(nvme-pci: Sync queues on reset)
|
|
|
|
Patchset: suspend
|
|
---
|
|
drivers/nvme/host/core.c | 36 ++++++++++++--
|
|
drivers/nvme/host/nvme.h | 7 +++
|
|
drivers/nvme/host/pci.c | 103 +++++++++++++++++++++++++++++++++++++--
|
|
drivers/pci/pcie/aspm.c | 20 ++++++++
|
|
include/linux/pci.h | 2 +
|
|
5 files changed, 162 insertions(+), 6 deletions(-)
|
|
|
|
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
|
|
index e64310f2296f..2a46419c5820 100644
|
|
--- a/drivers/nvme/host/core.c
|
|
+++ b/drivers/nvme/host/core.c
|
|
@@ -1065,15 +1065,15 @@ static struct nvme_id_ns *nvme_identify_ns(struct nvme_ctrl *ctrl,
|
|
return id;
|
|
}
|
|
|
|
-static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
|
|
- void *buffer, size_t buflen, u32 *result)
|
|
+static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
|
|
+ unsigned int dword11, void *buffer, size_t buflen, u32 *result)
|
|
{
|
|
union nvme_result res = { 0 };
|
|
struct nvme_command c;
|
|
int ret;
|
|
|
|
memset(&c, 0, sizeof(c));
|
|
- c.features.opcode = nvme_admin_set_features;
|
|
+ c.features.opcode = op;
|
|
c.features.fid = cpu_to_le32(fid);
|
|
c.features.dword11 = cpu_to_le32(dword11);
|
|
|
|
@@ -1084,6 +1084,24 @@ static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword
|
|
return ret;
|
|
}
|
|
|
|
+int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
|
|
+ unsigned int dword11, void *buffer, size_t buflen,
|
|
+ u32 *result)
|
|
+{
|
|
+ return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
|
|
+ buflen, result);
|
|
+}
|
|
+EXPORT_SYMBOL_GPL(nvme_set_features);
|
|
+
|
|
+int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
|
|
+ unsigned int dword11, void *buffer, size_t buflen,
|
|
+ u32 *result)
|
|
+{
|
|
+ return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
|
|
+ buflen, result);
|
|
+}
|
|
+EXPORT_SYMBOL_GPL(nvme_get_features);
|
|
+
|
|
int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
|
|
{
|
|
u32 q_count = (*count - 1) | ((*count - 1) << 16);
|
|
@@ -3630,6 +3648,18 @@ static void nvme_free_ctrl(struct device *dev)
|
|
nvme_put_subsystem(subsys);
|
|
}
|
|
|
|
+
|
|
+void nvme_sync_queues(struct nvme_ctrl *ctrl)
|
|
+{
|
|
+ struct nvme_ns *ns;
|
|
+
|
|
+ down_read(&ctrl->namespaces_rwsem);
|
|
+ list_for_each_entry(ns, &ctrl->namespaces, list)
|
|
+ blk_sync_queue(ns->queue);
|
|
+ up_read(&ctrl->namespaces_rwsem);
|
|
+}
|
|
+EXPORT_SYMBOL_GPL(nvme_sync_queues);
|
|
+
|
|
/*
|
|
* Initialize a NVMe controller structures. This needs to be called during
|
|
* earliest initialization so that we have the initialized structured around
|
|
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
|
|
index 276975506709..29e062d29c04 100644
|
|
--- a/drivers/nvme/host/nvme.h
|
|
+++ b/drivers/nvme/host/nvme.h
|
|
@@ -437,6 +437,7 @@ void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
|
|
void nvme_stop_queues(struct nvme_ctrl *ctrl);
|
|
void nvme_start_queues(struct nvme_ctrl *ctrl);
|
|
void nvme_kill_queues(struct nvme_ctrl *ctrl);
|
|
+void nvme_sync_queues(struct nvme_ctrl *ctrl);
|
|
void nvme_unfreeze(struct nvme_ctrl *ctrl);
|
|
void nvme_wait_freeze(struct nvme_ctrl *ctrl);
|
|
void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
|
|
@@ -454,6 +455,12 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
|
|
union nvme_result *result, void *buffer, unsigned bufflen,
|
|
unsigned timeout, int qid, int at_head,
|
|
blk_mq_req_flags_t flags);
|
|
+int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
|
|
+ unsigned int dword11, void *buffer, size_t buflen,
|
|
+ u32 *result);
|
|
+int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
|
|
+ unsigned int dword11, void *buffer, size_t buflen,
|
|
+ u32 *result);
|
|
int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count);
|
|
void nvme_stop_keep_alive(struct nvme_ctrl *ctrl);
|
|
int nvme_reset_ctrl(struct nvme_ctrl *ctrl);
|
|
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
|
|
index d7cf3202cdd3..bdafe953c9a9 100644
|
|
--- a/drivers/nvme/host/pci.c
|
|
+++ b/drivers/nvme/host/pci.c
|
|
@@ -26,6 +26,7 @@
|
|
#include <linux/mutex.h>
|
|
#include <linux/once.h>
|
|
#include <linux/pci.h>
|
|
+#include <linux/suspend.h>
|
|
#include <linux/t10-pi.h>
|
|
#include <linux/types.h>
|
|
#include <linux/io-64-nonatomic-lo-hi.h>
|
|
@@ -106,6 +107,7 @@ struct nvme_dev {
|
|
u32 cmbloc;
|
|
struct nvme_ctrl ctrl;
|
|
struct completion ioq_wait;
|
|
+ u32 last_ps;
|
|
|
|
mempool_t *iod_mempool;
|
|
|
|
@@ -2286,6 +2288,7 @@ static void nvme_reset_work(struct work_struct *work)
|
|
*/
|
|
if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
|
|
nvme_dev_disable(dev, false);
|
|
+ nvme_sync_queues(&dev->ctrl);
|
|
|
|
mutex_lock(&dev->shutdown_lock);
|
|
result = nvme_pci_enable(dev);
|
|
@@ -2624,16 +2627,101 @@ static void nvme_remove(struct pci_dev *pdev)
|
|
}
|
|
|
|
#ifdef CONFIG_PM_SLEEP
|
|
+static int nvme_get_power_state(struct nvme_ctrl *ctrl, u32 *ps)
|
|
+{
|
|
+ return nvme_get_features(ctrl, NVME_FEAT_POWER_MGMT, 0, NULL, 0, ps);
|
|
+}
|
|
+
|
|
+static int nvme_set_power_state(struct nvme_ctrl *ctrl, u32 ps)
|
|
+{
|
|
+ return nvme_set_features(ctrl, NVME_FEAT_POWER_MGMT, ps, NULL, 0, NULL);
|
|
+}
|
|
+
|
|
+static int nvme_resume(struct device *dev)
|
|
+{
|
|
+ struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
|
|
+ struct nvme_ctrl *ctrl = &ndev->ctrl;
|
|
+
|
|
+ if (ndev->last_ps == U32_MAX ||
|
|
+ nvme_set_power_state(ctrl, ndev->last_ps) != 0)
|
|
+ nvme_reset_ctrl(ctrl);
|
|
+ return 0;
|
|
+}
|
|
+
|
|
static int nvme_suspend(struct device *dev)
|
|
{
|
|
struct pci_dev *pdev = to_pci_dev(dev);
|
|
struct nvme_dev *ndev = pci_get_drvdata(pdev);
|
|
+ struct nvme_ctrl *ctrl = &ndev->ctrl;
|
|
+ int ret = -EBUSY;
|
|
+
|
|
+ ndev->last_ps = U32_MAX;
|
|
+
|
|
+ /*
|
|
+ * The platform does not remove power for a kernel managed suspend so
|
|
+ * use host managed nvme power settings for lowest idle power if
|
|
+ * possible. This should have quicker resume latency than a full device
|
|
+ * shutdown. But if the firmware is involved after the suspend or the
|
|
+ * device does not support any non-default power states, shut down the
|
|
+ * device fully.
|
|
+ *
|
|
+ * If ASPM is not enabled for the device, shut down the device and allow
|
|
+ * the PCI bus layer to put it into D3 in order to take the PCIe link
|
|
+ * down, so as to allow the platform to achieve its minimum low-power
|
|
+ * state (which may not be possible if the link is up).
|
|
+ */
|
|
+ if (pm_suspend_via_firmware() || !ctrl->npss ||
|
|
+ !pcie_aspm_enabled(pdev)) {
|
|
+ nvme_dev_disable(ndev, true);
|
|
+ return 0;
|
|
+ }
|
|
+
|
|
+ nvme_start_freeze(ctrl);
|
|
+ nvme_wait_freeze(ctrl);
|
|
+ nvme_sync_queues(ctrl);
|
|
+
|
|
+ if (ctrl->state != NVME_CTRL_LIVE &&
|
|
+ ctrl->state != NVME_CTRL_ADMIN_ONLY)
|
|
+ goto unfreeze;
|
|
+
|
|
+ ret = nvme_get_power_state(ctrl, &ndev->last_ps);
|
|
+ if (ret < 0)
|
|
+ goto unfreeze;
|
|
+
|
|
+ ret = nvme_set_power_state(ctrl, ctrl->npss);
|
|
+ if (ret < 0)
|
|
+ goto unfreeze;
|
|
+
|
|
+ if (ret) {
|
|
+ /*
|
|
+ * Clearing npss forces a controller reset on resume. The
|
|
+ * correct value will be resdicovered then.
|
|
+ */
|
|
+ nvme_dev_disable(ndev, true);
|
|
+ ctrl->npss = 0;
|
|
+ ret = 0;
|
|
+ goto unfreeze;
|
|
+ }
|
|
+ /*
|
|
+ * A saved state prevents pci pm from generically controlling the
|
|
+ * device's power. If we're using protocol specific settings, we don't
|
|
+ * want pci interfering.
|
|
+ */
|
|
+ pci_save_state(pdev);
|
|
+unfreeze:
|
|
+ nvme_unfreeze(ctrl);
|
|
+ return ret;
|
|
+}
|
|
+
|
|
+static int nvme_simple_suspend(struct device *dev)
|
|
+{
|
|
+ struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
|
|
|
|
nvme_dev_disable(ndev, true);
|
|
return 0;
|
|
}
|
|
|
|
-static int nvme_resume(struct device *dev)
|
|
+static int nvme_simple_resume(struct device *dev)
|
|
{
|
|
struct pci_dev *pdev = to_pci_dev(dev);
|
|
struct nvme_dev *ndev = pci_get_drvdata(pdev);
|
|
@@ -2641,9 +2729,16 @@ static int nvme_resume(struct device *dev)
|
|
nvme_reset_ctrl(&ndev->ctrl);
|
|
return 0;
|
|
}
|
|
-#endif
|
|
|
|
-static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
|
|
+const struct dev_pm_ops nvme_dev_pm_ops = {
|
|
+ .suspend = nvme_suspend,
|
|
+ .resume = nvme_resume,
|
|
+ .freeze = nvme_simple_suspend,
|
|
+ .thaw = nvme_simple_resume,
|
|
+ .poweroff = nvme_simple_suspend,
|
|
+ .restore = nvme_simple_resume,
|
|
+};
|
|
+#endif /* CONFIG_PM_SLEEP */
|
|
|
|
static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
|
|
pci_channel_state_t state)
|
|
@@ -2748,9 +2843,11 @@ static struct pci_driver nvme_driver = {
|
|
.probe = nvme_probe,
|
|
.remove = nvme_remove,
|
|
.shutdown = nvme_shutdown,
|
|
+#ifdef CONFIG_PM_SLEEP
|
|
.driver = {
|
|
.pm = &nvme_dev_pm_ops,
|
|
},
|
|
+#endif
|
|
.sriov_configure = pci_sriov_configure_simple,
|
|
.err_handler = &nvme_err_handler,
|
|
};
|
|
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
|
|
index 279f9f0197b0..cb474338f39d 100644
|
|
--- a/drivers/pci/pcie/aspm.c
|
|
+++ b/drivers/pci/pcie/aspm.c
|
|
@@ -1171,6 +1171,26 @@ static int pcie_aspm_get_policy(char *buffer, const struct kernel_param *kp)
|
|
module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
|
|
NULL, 0644);
|
|
|
|
+/**
|
|
+ * pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device.
|
|
+ * @pdev: Target device.
|
|
+ */
|
|
+bool pcie_aspm_enabled(struct pci_dev *pdev)
|
|
+{
|
|
+ struct pci_dev *bridge = pci_upstream_bridge(pdev);
|
|
+ bool ret;
|
|
+
|
|
+ if (!bridge)
|
|
+ return false;
|
|
+
|
|
+ mutex_lock(&aspm_lock);
|
|
+ ret = bridge->link_state ? !!bridge->link_state->aspm_enabled : false;
|
|
+ mutex_unlock(&aspm_lock);
|
|
+
|
|
+ return ret;
|
|
+}
|
|
+EXPORT_SYMBOL_GPL(pcie_aspm_enabled);
|
|
+
|
|
#ifdef CONFIG_PCIEASPM_DEBUG
|
|
static ssize_t link_state_show(struct device *dev,
|
|
struct device_attribute *attr,
|
|
diff --git a/include/linux/pci.h b/include/linux/pci.h
|
|
index 2fda9893962d..ec6c48ecd7d5 100644
|
|
--- a/include/linux/pci.h
|
|
+++ b/include/linux/pci.h
|
|
@@ -1488,8 +1488,10 @@ extern bool pcie_ports_native;
|
|
|
|
#ifdef CONFIG_PCIEASPM
|
|
bool pcie_aspm_support_enabled(void);
|
|
+bool pcie_aspm_enabled(struct pci_dev *pdev);
|
|
#else
|
|
static inline bool pcie_aspm_support_enabled(void) { return false; }
|
|
+static inline bool pcie_aspm_enabled(struct pci_dev *pdev) { return false; }
|
|
#endif
|
|
|
|
#ifdef CONFIG_PCIEAER
|
|
--
|
|
2.33.0
|
|
|